From 86ff2c6f66f59f170371bf407101214d89188a0c Mon Sep 17 00:00:00 2001 From: Alex Hope-O'Connor Date: Sun, 7 Dec 2025 10:47:20 +1000 Subject: [PATCH] Initial commit: BOM Local Service --- .dockerignore | 27 ++ .gitignore | 78 +++++ BomLocalService.csproj | 14 + Controllers/RadarController.cs | 163 +++++++++ Dockerfile | 46 +++ Models/CacheUpdateStatus.cs | 56 ++++ Models/LastUpdatedInfo.cs | 39 +++ Models/RadarScreenshotResponse.cs | 55 +++ Program.cs | 51 +++ Properties/launchSettings.json | 23 ++ Services/BomRadarService.cs | 217 ++++++++++++ Services/BrowserService.cs | 368 ++++++++++++++++++++ Services/CacheCleanupService.cs | 141 ++++++++ Services/CacheManagementService.cs | 210 ++++++++++++ Services/CacheService.cs | 189 +++++++++++ Services/DebugService.cs | 176 ++++++++++ Services/Interfaces/IBomRadarService.cs | 60 ++++ Services/Interfaces/IBrowserService.cs | 44 +++ Services/Interfaces/ICacheService.cs | 67 ++++ Services/Interfaces/IDebugService.cs | 72 ++++ Services/Interfaces/IScrapingService.cs | 34 ++ Services/Interfaces/ITimeParsingService.cs | 29 ++ Services/ScrapingService.cs | 373 +++++++++++++++++++++ Services/TimeParsingService.cs | 254 ++++++++++++++ Utilities/FilePathHelper.cs | 40 +++ Utilities/LocationHelper.cs | 63 ++++ Utilities/ResponseBuilder.cs | 40 +++ Utilities/SelectorHelper.cs | 37 ++ Utilities/StateAbbreviationHelper.cs | 45 +++ Utilities/ValidationHelper.cs | 24 ++ appsettings.json | 30 ++ docker-compose.yml | 45 +++ 32 files changed, 3110 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 BomLocalService.csproj create mode 100644 Controllers/RadarController.cs create mode 100644 Dockerfile create mode 100644 Models/CacheUpdateStatus.cs create mode 100644 Models/LastUpdatedInfo.cs create mode 100644 Models/RadarScreenshotResponse.cs create mode 100644 Program.cs create mode 100644 Properties/launchSettings.json create mode 100644 Services/BomRadarService.cs create mode 100644 Services/BrowserService.cs create mode 100644 Services/CacheCleanupService.cs create mode 100644 Services/CacheManagementService.cs create mode 100644 Services/CacheService.cs create mode 100644 Services/DebugService.cs create mode 100644 Services/Interfaces/IBomRadarService.cs create mode 100644 Services/Interfaces/IBrowserService.cs create mode 100644 Services/Interfaces/ICacheService.cs create mode 100644 Services/Interfaces/IDebugService.cs create mode 100644 Services/Interfaces/IScrapingService.cs create mode 100644 Services/Interfaces/ITimeParsingService.cs create mode 100644 Services/ScrapingService.cs create mode 100644 Services/TimeParsingService.cs create mode 100644 Utilities/FilePathHelper.cs create mode 100644 Utilities/LocationHelper.cs create mode 100644 Utilities/ResponseBuilder.cs create mode 100644 Utilities/SelectorHelper.cs create mode 100644 Utilities/StateAbbreviationHelper.cs create mode 100644 Utilities/ValidationHelper.cs create mode 100644 appsettings.json create mode 100644 docker-compose.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e96dd6c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,27 @@ +**/.classpath +**/.dockerignore +**/.env +**/.git +**/.gitignore +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/.idea +**/*.*proj.user +**/*.dbmdl +**/*.jfm +**/azds.yaml +**/bin +**/charts +**/docker-compose* +**/Dockerfile* +**/node_modules +**/npm-debug.log +**/obj +**/secrets.dev.yaml +**/values.dev.yaml +LICENSE +README.md + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6106b03 --- /dev/null +++ b/.gitignore @@ -0,0 +1,78 @@ +## .NET build artifacts +bin/ +obj/ +*.dll +*.exe +*.pdb +*.cache +*.deps.json +*.runtimeconfig.json +*.staticwebassets.endpoints.json + +## Build results +[Dd]ebug/ +[Rr]elease/ +[Bb]uild/ +x64/ +x86/ +[Aa]rm/ +[Aa]rm64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +## Visual Studio / Rider +.vs/ +.idea/ +*.user +*.suo +*.userosscache +*.sln.docstates +*.userprefs + +## VS Code +.vscode/ +*.code-workspace + +## Cache directory (generated screenshots and metadata) +cache/ + +## NuGet +*.nupkg +*.snupkg +**/packages/* +!**/packages/build/ +project.lock.json +project.fragment.lock.json +artifacts/ + +## Test results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* +*.trx +*.coverage +*.coveragexml + +## OS files +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db +Desktop.ini + +## Temporary files +*.tmp +*.temp +*.swp +*~ +.*.sw? + +## Docker +# Keep Dockerfile and docker-compose.yml, ignore temporary Docker files +*.dockerignore.bak + diff --git a/BomLocalService.csproj b/BomLocalService.csproj new file mode 100644 index 0000000..48826b1 --- /dev/null +++ b/BomLocalService.csproj @@ -0,0 +1,14 @@ + + + + net9.0 + enable + enable + + + + + + + + diff --git a/Controllers/RadarController.cs b/Controllers/RadarController.cs new file mode 100644 index 0000000..abb490b --- /dev/null +++ b/Controllers/RadarController.cs @@ -0,0 +1,163 @@ +using BomLocalService.Models; +using BomLocalService.Services.Interfaces; +using BomLocalService.Utilities; +using Microsoft.AspNetCore.Mvc; + +namespace BomLocalService.Controllers; + +[ApiController] +[Route("api/radar")] +public class RadarController : ControllerBase +{ + private readonly IBomRadarService _bomRadarService; + private readonly ILogger _logger; + + public RadarController(IBomRadarService bomRadarService, ILogger logger) + { + _bomRadarService = bomRadarService; + _logger = logger; + } + + /// + /// Get radar screenshot for a location. Returns cached screenshot if available, or triggers background update. + /// + [HttpGet("{suburb}/{state}")] + public async Task GetScreenshot(string suburb, string state, CancellationToken cancellationToken = default) + { + try + { + var validationError = ValidationHelper.ValidateLocation(suburb, state); + if (validationError != null) + { + return BadRequest(new { error = validationError }); + } + + // Check if cache needs updating and trigger if needed (non-blocking) + _ = Task.Run(async () => + { + try + { + await _bomRadarService.TriggerCacheUpdateAsync(suburb, state, CancellationToken.None); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Background cache update failed for {Suburb}, {State}", suburb, state); + } + }); + + // Return cached screenshot if it exists (no waiting) + var result = await _bomRadarService.GetCachedScreenshotAsync(suburb, state, cancellationToken); + + if (result == null) + { + return NotFound(new { + error = "Screenshot not found in cache. Cache update has been triggered in background. Please retry in a few moments.", + retryAfter = 30, // seconds + refreshEndpoint = $"/api/radar/{suburb}/{state}/refresh" + }); + } + + // Return the image file + if (System.IO.File.Exists(result.ImagePath)) + { + var imageBytes = await System.IO.File.ReadAllBytesAsync(result.ImagePath, cancellationToken); + var contentType = "image/png"; + return File(imageBytes, contentType, Path.GetFileName(result.ImagePath)); + } + + return NotFound(new { error = "Screenshot file not found" }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error getting cached radar screenshot for suburb: {Suburb}, state: {State}", suburb, state); + return StatusCode(500, new { error = "An error occurred while getting the radar screenshot", message = ex.Message }); + } + } + + /// + /// Get metadata about the cached radar data for a location (observation time, forecast time, etc.) + /// + [HttpGet("{suburb}/{state}/metadata")] + public async Task> GetMetadata(string suburb, string state, CancellationToken cancellationToken = default) + { + try + { + var validationError = ValidationHelper.ValidateLocation(suburb, state); + if (validationError != null) + { + return BadRequest(new { error = validationError }); + } + + var result = await _bomRadarService.GetLastUpdatedInfoAsync(suburb, state, cancellationToken); + + if (result == null) + { + return NotFound(new { error = "No cached data found. Use POST /api/radar/{suburb}/{state}/refresh to trigger cache update." }); + } + + return Ok(result); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error getting metadata for suburb: {Suburb}, state: {State}", suburb, state); + return StatusCode(500, new { error = "An error occurred while getting metadata", message = ex.Message }); + } + } + + /// + /// Manually trigger a cache refresh for a location. Returns status of the update operation. + /// + [HttpPost("{suburb}/{state}/refresh")] + public async Task> RefreshCache(string suburb, string state, CancellationToken cancellationToken = default) + { + try + { + var validationError = ValidationHelper.ValidateLocation(suburb, state); + if (validationError != null) + { + return BadRequest(new { error = validationError }); + } + + var status = await _bomRadarService.TriggerCacheUpdateAsync(suburb, state, cancellationToken); + return Ok(status); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error triggering cache update for suburb: {Suburb}, state: {State}", suburb, state); + return StatusCode(500, new { error = "An error occurred while triggering cache update", message = ex.Message }); + } + } + + /// + /// Delete cached data for a location. + /// + [HttpDelete("{suburb}/{state}")] + public async Task DeleteCache(string suburb, string state, CancellationToken cancellationToken = default) + { + try + { + var validationError = ValidationHelper.ValidateLocation(suburb, state); + if (validationError != null) + { + return BadRequest(new { error = validationError }); + } + + var deleted = await _bomRadarService.DeleteCachedLocationAsync(suburb, state, cancellationToken); + + if (deleted) + { + return Ok(new { message = $"Cache deleted for {suburb}, {state}" }); + } + else + { + return NotFound(new { error = $"No cached data found for {suburb}, {state}" }); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Error deleting cached location for suburb: {Suburb}, state: {State}", suburb, state); + return StatusCode(500, new { error = "An error occurred while deleting cached location", message = ex.Message }); + } + } +} + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8248c26 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,46 @@ +# Build stage +FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build +WORKDIR /src + +# Copy csproj and restore dependencies +COPY ["BomLocalService.csproj", "./"] +RUN dotnet restore "BomLocalService.csproj" + +# Copy everything else and build +COPY . . +RUN dotnet build "BomLocalService.csproj" -c Release -o /app/build + +# Publish stage +FROM build AS publish +RUN dotnet publish "BomLocalService.csproj" -c Release -o /app/publish /p:UseAppHost=false + +# Runtime stage - use official Playwright .NET image that has browsers and dependencies pre-installed +FROM mcr.microsoft.com/playwright/dotnet:v1.57.0-noble AS final +WORKDIR /app + +# Install .NET 9 runtime and Xvfb for virtual display (Playwright image has browsers but may need .NET 9) +RUN apt-get update && \ + wget https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh && \ + chmod +x dotnet-install.sh && \ + ./dotnet-install.sh --channel 9.0 --install-dir /usr/share/dotnet && \ + rm dotnet-install.sh && \ + apt-get install -y xvfb x11vnc fluxbox && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Copy published app +COPY --from=publish /app/publish . + +# Create cache directory for screenshots +RUN mkdir -p /app/cache && chmod 777 /app/cache + +# Create startup script to run Xvfb and then the app +RUN echo '#!/bin/bash\n\ +Xvfb :99 -screen 0 1920x1080x24 > /dev/null 2>&1 &\n\ +export DISPLAY=:99\n\ +exec dotnet BomLocalService.dll' > /app/start.sh && \ + chmod +x /app/start.sh + +EXPOSE 8080 + +ENTRYPOINT ["/app/start.sh"] diff --git a/Models/CacheUpdateStatus.cs b/Models/CacheUpdateStatus.cs new file mode 100644 index 0000000..77e90cc --- /dev/null +++ b/Models/CacheUpdateStatus.cs @@ -0,0 +1,56 @@ +namespace BomLocalService.Models; + +/// +/// Status information about a cache update operation for a location. +/// Returned when manually triggering a cache refresh via the refresh endpoint. +/// +public class CacheUpdateStatus +{ + /// + /// Indicates whether a background cache update was triggered. + /// True if the cache was missing or stale and an update was initiated. + /// False if the cache is valid and no update was needed. + /// + public bool UpdateTriggered { get; set; } + + /// + /// Indicates whether cached screenshot files exist for this location. + /// True if PNG screenshot files are found in the cache directory. + /// False if no cached files exist for this location. + /// + public bool CacheExists { get; set; } + + /// + /// Indicates whether the existing cache is still valid (not expired). + /// Cache is considered valid if the observation time plus expiration buffer (typically 15.5 minutes) + /// is still in the future. BOM updates observations every 15 minutes, so cache expiration + /// includes a small buffer to account for timing variations. + /// True if cache exists and is still valid, false if cache is missing or expired. + /// + public bool CacheIsValid { get; set; } + + /// + /// The UTC timestamp when the current cache will expire and should be refreshed. + /// Calculated as: ObservationTime + CacheExpirationMinutes (typically observation time + 15.5 minutes). + /// Null if no cache exists or metadata is not available. + /// + public DateTime? CacheExpiresAt { get; set; } + + /// + /// The UTC timestamp when the next cache update should occur. + /// If cache is valid: equals CacheExpiresAt (when current cache expires). + /// If cache is invalid/missing: equals current time + CacheExpirationMinutes (when background update will complete). + /// Null if cache is valid and CacheExpiresAt is null. + /// + public DateTime? NextUpdateTime { get; set; } + + /// + /// Human-readable message describing the cache status and action taken. + /// Possible values: + /// - "Cache is valid, no update needed" - Cache exists and is still fresh + /// - "Cache is stale, update triggered" - Cache exists but expired, update initiated + /// - "No cache exists, update triggered" - No cache found, update initiated + /// + public string? Message { get; set; } +} + diff --git a/Models/LastUpdatedInfo.cs b/Models/LastUpdatedInfo.cs new file mode 100644 index 0000000..5d68dae --- /dev/null +++ b/Models/LastUpdatedInfo.cs @@ -0,0 +1,39 @@ +namespace BomLocalService.Models; + +/// +/// Metadata extracted from the BOM weather map page about observation and forecast times. +/// This information is scraped from the weather metadata section on the BOM website. +/// +public class LastUpdatedInfo +{ + /// + /// The UTC timestamp when the weather observations were taken. + /// Extracted from the BOM website metadata section (e.g., "Observations: 11 minutes ago, 8:20 pm AEST"). + /// BOM updates observations every 15 minutes (at :00, :15, :30, :45 past the hour). + /// + public DateTime ObservationTime { get; set; } + + /// + /// The UTC timestamp when the weather forecast was generated. + /// Extracted from the BOM website metadata section (e.g., "Forecast: 41 minutes ago, 7:50 pm AEST"). + /// Forecasts are typically updated less frequently than observations. + /// + public DateTime ForecastTime { get; set; } + + /// + /// The name of the weather station providing the observations for this location. + /// Extracted from text like "at Gympie weather station" in the metadata section. + /// May be null if the station name cannot be parsed from the page. + /// Example: "Gympie", "Brisbane", "Sydney" + /// + public string? WeatherStation { get; set; } + + /// + /// The distance from the requested location to the weather station. + /// Extracted from text like "30 km from Pomona, QLD" in the metadata section. + /// Format: "{number} km" (e.g., "30 km", "15 km"). + /// May be null if the distance cannot be parsed from the page. + /// + public string? Distance { get; set; } +} + diff --git a/Models/RadarScreenshotResponse.cs b/Models/RadarScreenshotResponse.cs new file mode 100644 index 0000000..043d31c --- /dev/null +++ b/Models/RadarScreenshotResponse.cs @@ -0,0 +1,55 @@ +namespace BomLocalService.Models; + +/// +/// Response model containing the radar screenshot image path and associated metadata. +/// This is the primary response returned when requesting a radar screenshot for a location. +/// +public class RadarScreenshotResponse +{ + /// + /// The full file system path to the cached PNG screenshot image file. + /// This is the path on the server where the screenshot is stored. + /// Format: "{CacheDirectory}/{Suburb}_{State}_{Timestamp}.png" + /// Example: "/app/cache/Pomona_QLD_20251207_000906.png" + /// + public string ImagePath { get; set; } = string.Empty; + + /// + /// The UTC timestamp when the screenshot file was last written/modified on disk. + /// This is the file system modification time, which typically matches when the screenshot was captured. + /// Used as a fallback if metadata is not available. + /// + public DateTime LastUpdated { get; set; } + + /// + /// The UTC timestamp when the weather observations were taken (from BOM website). + /// This comes from the LastUpdatedInfo metadata extracted from the BOM weather map page. + /// BOM updates observations every 15 minutes (at :00, :15, :30, :45 past the hour). + /// If metadata is not available, defaults to DateTime.UtcNow. + /// + public DateTime ObservationTime { get; set; } + + /// + /// The UTC timestamp when the weather forecast was generated (from BOM website). + /// This comes from the LastUpdatedInfo metadata extracted from the BOM weather map page. + /// Forecasts are typically updated less frequently than observations. + /// If metadata is not available, defaults to DateTime.UtcNow. + /// + public DateTime ForecastTime { get; set; } + + /// + /// The name of the weather station providing the observations for this location. + /// Extracted from the BOM website metadata (e.g., "Gympie", "Brisbane"). + /// May be null if the station name cannot be parsed or if metadata is not available. + /// + public string? WeatherStation { get; set; } + + /// + /// The distance from the requested location to the weather station. + /// Extracted from the BOM website metadata (e.g., "30 km", "15 km"). + /// Format: "{number} km". + /// May be null if the distance cannot be parsed or if metadata is not available. + /// + public string? Distance { get; set; } +} + diff --git a/Program.cs b/Program.cs new file mode 100644 index 0000000..378d1fd --- /dev/null +++ b/Program.cs @@ -0,0 +1,51 @@ +using BomLocalService.Services; +using BomLocalService.Services.Interfaces; + +var builder = WebApplication.CreateBuilder(args); + +// Add services to the container +builder.Services.AddControllers(); +builder.Services.AddEndpointsApiExplorer(); +builder.Services.AddOpenApi(); + +// Register core services via interfaces (order matters - dependencies must be registered first) +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); + +// Register BOM Radar Service as singleton (orchestrator, depends on all above services) +builder.Services.AddSingleton(); + +// Register background services (order matters - management service needs radar service) +builder.Services.AddHostedService(); +builder.Services.AddHostedService(); + +// Add configuration - appsettings.json provides default values +// All values can be overridden via environment variables +builder.Configuration.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true); + +var app = builder.Build(); + +// Configure the HTTP request pipeline +if (app.Environment.IsDevelopment()) +{ + app.MapOpenApi(); +} + +app.UseHttpsRedirection(); +app.UseAuthorization(); +app.MapControllers(); + +// Cleanup on shutdown +app.Lifetime.ApplicationStopped.Register(() => +{ + var service = app.Services.GetService(); + if (service is IDisposable disposable) + { + disposable.Dispose(); + } +}); + +app.Run(); diff --git a/Properties/launchSettings.json b/Properties/launchSettings.json new file mode 100644 index 0000000..307dd52 --- /dev/null +++ b/Properties/launchSettings.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": false, + "applicationUrl": "http://localhost:5249", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": false, + "applicationUrl": "https://localhost:7190;http://localhost:5249", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/Services/BomRadarService.cs b/Services/BomRadarService.cs new file mode 100644 index 0000000..d20232e --- /dev/null +++ b/Services/BomRadarService.cs @@ -0,0 +1,217 @@ +using BomLocalService.Models; +using BomLocalService.Services.Interfaces; +using BomLocalService.Utilities; +using Microsoft.Playwright; + +namespace BomLocalService.Services; + +public class BomRadarService : IBomRadarService, IDisposable +{ + private readonly ILogger _logger; + private readonly ICacheService _cacheService; + private readonly IBrowserService _browserService; + private readonly IScrapingService _scrapingService; + private readonly IDebugService _debugService; + private readonly double _cacheExpirationMinutes; + + public BomRadarService( + ILogger logger, + ICacheService cacheService, + IBrowserService browserService, + IScrapingService scrapingService, + IDebugService debugService, + IConfiguration configuration) + { + _logger = logger; + _cacheService = cacheService; + _browserService = browserService; + _scrapingService = scrapingService; + _debugService = debugService; + _cacheExpirationMinutes = configuration.GetValue("CacheExpirationMinutes", 15.5); + } + + public async Task GetCachedScreenshotAsync(string suburb, string state, CancellationToken cancellationToken = default) + { + var (cachedPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken); + + if (string.IsNullOrEmpty(cachedPath) || !File.Exists(cachedPath)) + { + return null; + } + + return ResponseBuilder.CreateRadarScreenshotResponse(cachedPath, cachedMetadata); + } + + public async Task TriggerCacheUpdateAsync(string suburb, string state, CancellationToken cancellationToken = default) + { + var status = new CacheUpdateStatus(); + var (cachedPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken); + + status.CacheExists = !string.IsNullOrEmpty(cachedPath) && File.Exists(cachedPath); + + if (cachedMetadata != null) + { + status.CacheIsValid = _cacheService.IsCacheValid(cachedMetadata); + status.CacheExpiresAt = cachedMetadata.ObservationTime.AddMinutes(_cacheExpirationMinutes); + } + + // Check if we need to update + bool needsUpdate = !status.CacheExists || !status.CacheIsValid; + + if (needsUpdate) + { + // Trigger async update (fire and forget) + _ = Task.Run(async () => + { + try + { + await FetchAndCacheScreenshotAsync(suburb, state, CancellationToken.None); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error during background cache update for {Suburb}, {State}", suburb, state); + } + }, cancellationToken); + + status.UpdateTriggered = true; + status.Message = status.CacheExists + ? "Cache is stale, update triggered" + : "No cache exists, update triggered"; + + // Calculate next update time + if (status.CacheExpiresAt.HasValue && status.CacheExpiresAt.Value > DateTime.UtcNow) + { + status.NextUpdateTime = status.CacheExpiresAt.Value; + } + else + { + status.NextUpdateTime = DateTime.UtcNow.AddMinutes(_cacheExpirationMinutes); + } + } + else + { + status.UpdateTriggered = false; + status.Message = "Cache is valid, no update needed"; + status.NextUpdateTime = status.CacheExpiresAt; + } + + return status; + } + + private async Task FetchAndCacheScreenshotAsync(string suburb, string state, CancellationToken cancellationToken = default) + { + _logger.LogInformation("Getting radar screenshot for suburb: {Suburb}, state: {State}", suburb, state); + + // Check cache FIRST, before acquiring semaphore (cached requests shouldn't block) + var (cachedPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken); + + if (!string.IsNullOrEmpty(cachedPath) && File.Exists(cachedPath) && cachedMetadata != null) + { + var isValid = _cacheService.IsCacheValid(cachedMetadata); + if (isValid) + { + _logger.LogInformation("Returning valid cached screenshot for {Suburb}, {State} (no semaphore needed)", suburb, state); + return ResponseBuilder.CreateRadarScreenshotResponse(cachedPath, cachedMetadata); + } + else + { + var nextUpdate = cachedMetadata.ObservationTime.AddMinutes(_cacheExpirationMinutes); + var timeUntilExpiry = nextUpdate - DateTime.UtcNow; + _logger.LogInformation("Cached screenshot exists but is stale (observation time: {ObservationTime}, expired {TimeAgo} ago), fetching new one", + cachedMetadata.ObservationTime, -timeUntilExpiry); + } + } + else if (!string.IsNullOrEmpty(cachedPath) && File.Exists(cachedPath)) + { + _logger.LogWarning("Cached screenshot found but no metadata file exists, fetching new one"); + } + else + { + _logger.LogInformation("No cached screenshot found for {Suburb}, {State}, fetching new one", suburb, state); + } + + // Only acquire semaphore if we need to fetch new screenshot + var semaphore = _browserService.GetSemaphore(); + await semaphore.WaitAsync(cancellationToken); + + // Generate request ID for debugging + var requestId = $"{DateTime.UtcNow:yyyyMMdd_HHmmss}_{Guid.NewGuid():N}"; + var debugFolder = _debugService.CreateRequestFolder(requestId); + + IBrowserContext? context = null; + try + { + // Double-check cache after acquiring semaphore (another request might have just created it) + var (recheckCachedPath, recheckCachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken); + if (!string.IsNullOrEmpty(recheckCachedPath) && File.Exists(recheckCachedPath) && recheckCachedMetadata != null && _cacheService.IsCacheValid(recheckCachedMetadata)) + { + _logger.LogInformation("Cache became valid while waiting for semaphore, returning cached screenshot"); + return ResponseBuilder.CreateRadarScreenshotResponse(recheckCachedPath, recheckCachedMetadata); + } + + // Need to capture new screenshot + context = await _browserService.CreateContextAsync(); + var (page, consoleMessages, networkRequests) = await _browserService.CreatePageWithDebugAsync(context, requestId); + + try + { + return await _scrapingService.ScrapeRadarScreenshotAsync( + suburb, + state, + debugFolder, + page, + consoleMessages, + networkRequests, + cancellationToken); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error during radar screenshot capture (RequestId: {RequestId})", requestId); + throw; + } + finally + { + await context.CloseAsync(); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Error getting radar screenshot for suburb: {Suburb}, state: {State} (RequestId: {RequestId})", suburb, state, requestId); + throw; + } + finally + { + semaphore.Release(); + } + } + + public async Task GetLastUpdatedInfoAsync(string suburb, string state, CancellationToken cancellationToken = default) + { + // Return cached metadata if available, otherwise return null + var (_, metadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken); + + if (metadata != null) + { + _logger.LogDebug("Returning cached last updated info for {Suburb}, {State}", suburb, state); + return metadata; + } + + _logger.LogDebug("No cached metadata found for {Suburb}, {State}", suburb, state); + return null; + } + + public Task GetCachedScreenshotPathAsync(string suburb, string state, CancellationToken cancellationToken = default) + { + return _cacheService.GetCachedScreenshotPathAsync(suburb, state, cancellationToken); + } + + public Task DeleteCachedLocationAsync(string suburb, string state, CancellationToken cancellationToken = default) + { + return _cacheService.DeleteCachedLocationAsync(suburb, state, cancellationToken); + } + + public void Dispose() + { + _browserService?.Dispose(); + } +} diff --git a/Services/BrowserService.cs b/Services/BrowserService.cs new file mode 100644 index 0000000..84780d2 --- /dev/null +++ b/Services/BrowserService.cs @@ -0,0 +1,368 @@ +using BomLocalService.Services.Interfaces; +using Microsoft.Playwright; + +namespace BomLocalService.Services; + +public class BrowserService : IBrowserService +{ + private readonly ILogger _logger; + private readonly string _timezone; + private readonly IDebugService _debugService; + private IPlaywright? _playwright; + private IBrowser? _browser; + private readonly SemaphoreSlim _semaphore = new(1, 1); + + public BrowserService(ILogger logger, IConfiguration configuration, IDebugService debugService) + { + _logger = logger; + _timezone = configuration.GetValue("Timezone") ?? "Australia/Brisbane"; + _debugService = debugService; + } + + /// + /// Gets or creates the Playwright instance + /// + private async Task GetPlaywrightAsync() + { + if (_playwright == null) + { + _playwright = await Microsoft.Playwright.Playwright.CreateAsync(); + } + return _playwright; + } + + /// + /// Gets or creates the browser instance + /// + public async Task GetBrowserAsync() + { + if (_browser == null) + { + var playwright = await GetPlaywrightAsync(); + _browser = await playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions + { + Headless = false, // Run in visible mode with virtual display (Xvfb) + Args = new[] + { + "--no-sandbox", + "--disable-setuid-sandbox", + "--disable-blink-features=AutomationControlled", + "--disable-dev-shm-usage", + "--disable-infobars", + "--disable-extensions", + "--disable-background-networking", + "--disable-background-timer-throttling", + "--disable-backgrounding-occluded-windows", + "--disable-breakpad", + "--disable-client-side-phishing-detection", + "--disable-component-extensions-with-background-pages", + "--disable-component-update", + "--disable-default-apps", + "--disable-domain-reliability", + "--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees", + "--disable-hang-monitor", + "--disable-ipc-flooding-protection", + "--disable-notifications", + "--disable-offer-store-unmasked-wallet-cards", + "--disable-popup-blocking", + "--disable-prompt-on-repost", + "--disable-renderer-backgrounding", + "--disable-sync", + "--force-color-profile=srgb", + "--metrics-recording-only", + "--no-first-run", + "--no-default-browser-check", + "--no-pings", + "--password-store=basic", + "--use-mock-keychain", + "--enable-automation=false", + "--exclude-switches=enable-automation", + "--disable-features=UserAgentClientHint", + "--enable-unsafe-swiftshader", // Required for software WebGL (Chrome 137+ deprecated automatic fallback) + "--use-gl=swiftshader" // Use SwiftShader for WebGL software rendering + } + }); + } + return _browser; + } + + /// + /// Creates a new browser context with proper configuration + /// + public async Task CreateContextAsync() + { + var browser = await GetBrowserAsync(); + return await browser.NewContextAsync(new BrowserNewContextOptions + { + ViewportSize = new ViewportSize { Width = 1920, Height = 1080 }, + UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + Locale = "en-AU", + TimezoneId = _timezone, + Permissions = new[] { "geolocation" }, + ExtraHTTPHeaders = new Dictionary + { + { "Accept-Language", "en-AU,en;q=0.9" }, + { "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8" }, + { "Accept-Encoding", "gzip, deflate, br" }, + { "Connection", "keep-alive" }, + { "Upgrade-Insecure-Requests", "1" }, + { "Sec-Fetch-Dest", "document" }, + { "Sec-Fetch-Mode", "navigate" }, + { "Sec-Fetch-Site", "none" }, + { "Cache-Control", "max-age=0" } + } + }); + } + + /// + /// Creates a new page with anti-detection scripts and debug event handlers + /// + public async Task<(IPage page, List<(string type, string text, DateTime timestamp)> consoleMessages, List<(string method, string url, int? status, string resourceType, DateTime timestamp)> networkRequests)> CreatePageWithDebugAsync( + IBrowserContext context, + string requestId) + { + var page = await context.NewPageAsync(); + + // Set up console message and network request capture for debugging + var consoleMessages = new List<(string type, string text, DateTime timestamp)>(); + var networkRequests = new List<(string method, string url, int? status, string resourceType, DateTime timestamp)>(); + + if (_debugService.IsEnabled) + { + // Capture page console messages (console.log, console.error, etc.) + page.Console += (sender, e) => + { + consoleMessages.Add((e.Type.ToString(), e.Text, DateTime.UtcNow)); + }; + + // Capture page errors (unhandled JavaScript errors) + page.PageError += (sender, e) => + { + consoleMessages.Add(("error", $"PageError: {e}", DateTime.UtcNow)); + }; + + // Capture context console messages (broader scope, may include more warnings) + context.Console += (sender, e) => + { + consoleMessages.Add((e.Type.ToString(), $"[Context] {e.Text}", DateTime.UtcNow)); + }; + + // Capture network requests + page.Request += (sender, e) => + { + networkRequests.Add((e.Method, e.Url, null, e.ResourceType, DateTime.UtcNow)); + }; + + // Capture failed requests (these often show up as console errors in browser) + page.RequestFailed += (sender, e) => + { + networkRequests.Add((e.Method, e.Url, null, e.ResourceType, DateTime.UtcNow)); + var failureText = e.Failure?.ToString() ?? "Unknown failure"; + consoleMessages.Add(("error", $"Request failed: {e.Method} {e.Url} - {failureText}", DateTime.UtcNow)); + }; + + page.Response += (sender, e) => + { + // Update the request with response status + var matchingRequests = networkRequests.Where(r => r.url == e.Request.Url && r.status == null).ToList(); + if (matchingRequests.Any()) + { + var request = matchingRequests.Last(); + var index = networkRequests.IndexOf(request); + networkRequests[index] = (request.method, request.url, e.Status, request.resourceType, request.timestamp); + } + }; + } + + // Inject comprehensive anti-detection scripts (must run before page load) + await page.AddInitScriptAsync(GetAntiDetectionScript()); + + return (page, consoleMessages, networkRequests); + } + + /// + /// Gets the anti-detection JavaScript script + /// + private static string GetAntiDetectionScript() + { + return @" + (function() { + 'use strict'; + + // Remove webdriver property completely + Object.defineProperty(navigator, 'webdriver', { + get: () => false, + configurable: true + }); + + // Delete it from prototype chain + try { + delete navigator.__proto__.webdriver; + } catch (e) {} + + // Override chrome object + window.chrome = { + runtime: {}, + loadTimes: function() {}, + csi: function() {}, + app: {} + }; + + // Override plugins with realistic data + Object.defineProperty(navigator, 'plugins', { + get: () => { + const plugins = []; + for (let i = 0; i < 3; i++) { + plugins.push({ + 0: { type: 'application/x-google-chrome-pdf', suffixes: 'pdf', description: 'Portable Document Format' }, + description: 'Portable Document Format', + filename: 'internal-pdf-viewer', + length: 1, + name: 'Chrome PDF Plugin' + }); + } + return plugins; + }, + configurable: true + }); + + // Override languages + Object.defineProperty(navigator, 'languages', { + get: () => ['en-AU', 'en', 'en-US'], + configurable: true + }); + + // Override language + Object.defineProperty(navigator, 'language', { + get: () => 'en-AU', + configurable: true + }); + + // Override vendor + Object.defineProperty(navigator, 'vendor', { + get: () => 'Google Inc.', + configurable: true + }); + + // Override platform + Object.defineProperty(navigator, 'platform', { + get: () => 'Win32', + configurable: true + }); + + // Override hardwareConcurrency + Object.defineProperty(navigator, 'hardwareConcurrency', { + get: () => 8, + configurable: true + }); + + // Override deviceMemory + if ('deviceMemory' in navigator) { + Object.defineProperty(navigator, 'deviceMemory', { + get: () => 8, + configurable: true + }); + } + + // Spoof connection - use named functions to avoid warnings + Object.defineProperty(navigator, 'connection', { + get: () => ({ + effectiveType: '4g', + rtt: 50, + downlink: 10, + saveData: false, + onchange: null, + addEventListener: function addEventListener() {}, + removeEventListener: function removeEventListener() {}, + dispatchEvent: function dispatchEvent() { return true; } + }), + configurable: true + }); + + // Override permissions API - bind context to avoid illegal invocation + const originalQuery = window.navigator.permissions.query.bind(window.navigator.permissions); + window.navigator.permissions.query = function query(parameters) { + return parameters.name === 'notifications' ? + Promise.resolve({ state: Notification.permission }) : + originalQuery(parameters); + }; + + // Override getBattery - bind context to avoid illegal invocation + if ('getBattery' in navigator) { + navigator.getBattery = function getBattery() { + return Promise.resolve({ + charging: true, + chargingTime: 0, + dischargingTime: Infinity, + level: 1, + onchargingchange: null, + onchargingtimechange: null, + ondischargingtimechange: null, + onlevelchange: null, + addEventListener: function addEventListener() {}, + removeEventListener: function removeEventListener() {}, + dispatchEvent: function dispatchEvent() { return true; } + }); + }; + } + + // DON'T use Proxy on navigator - it breaks APIs like getGamepads() + // Just hide webdriver property directly + try { + delete navigator.__proto__.webdriver; + } catch (e) {} + + // AudioContext fingerprinting + if (window.AudioContext) { + const originalAudioContext = window.AudioContext; + window.AudioContext = function() { + const ctx = new originalAudioContext(); + const originalCreateOscillator = ctx.createOscillator; + ctx.createOscillator = function() { + const oscillator = originalCreateOscillator.call(this); + const originalFrequency = Object.getOwnPropertyDescriptor( + AudioParam.prototype, 'value' + ).get; + Object.defineProperty(oscillator.frequency, 'value', { + get: originalFrequency, + set: function() {} + }); + return oscillator; + }; + return ctx; + }; + } + })(); + "; + } + + /// + /// Initializes the browser (pre-warming) + /// + public async Task InitializeAsync(CancellationToken cancellationToken = default) + { + _logger.LogInformation("Pre-warming browser"); + try + { + await GetBrowserAsync(); + _logger.LogInformation("Browser initialized and ready"); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to pre-warm browser, will initialize on first use"); + } + } + + /// + /// Gets the semaphore for synchronizing browser operations + /// + public SemaphoreSlim GetSemaphore() => _semaphore; + + public void Dispose() + { + _browser?.DisposeAsync().AsTask().Wait(); + _playwright?.Dispose(); + _semaphore?.Dispose(); + } +} + diff --git a/Services/CacheCleanupService.cs b/Services/CacheCleanupService.cs new file mode 100644 index 0000000..9486b01 --- /dev/null +++ b/Services/CacheCleanupService.cs @@ -0,0 +1,141 @@ +using BomLocalService.Utilities; +using Microsoft.Extensions.Hosting; + +namespace BomLocalService.Services; + +public class CacheCleanupService : BackgroundService +{ + private readonly ILogger _logger; + private readonly string _cacheDirectory; + private readonly int _retentionHours; + private readonly TimeSpan _cleanupInterval; + + public CacheCleanupService(ILogger logger, IConfiguration configuration) + { + _logger = logger; + _cacheDirectory = FilePathHelper.GetCacheDirectory(configuration); + _retentionHours = configuration.GetValue("CacheRetentionHours", 24); + var cleanupIntervalHours = configuration.GetValue("CacheCleanup:IntervalHours", 1); + _cleanupInterval = TimeSpan.FromHours(cleanupIntervalHours); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _logger.LogInformation("Cache cleanup service started. Retention: {RetentionHours} hours, Interval: {Interval}", + _retentionHours, _cleanupInterval); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + await CleanupOldCacheFilesAsync(stoppingToken); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error during cache cleanup"); + } + + await Task.Delay(_cleanupInterval, stoppingToken); + } + } + + private Task CleanupOldCacheFilesAsync(CancellationToken cancellationToken) + { + if (!Directory.Exists(_cacheDirectory)) + { + return Task.CompletedTask; + } + + var cutoffTime = DateTime.UtcNow.AddHours(-_retentionHours); + var deletedCount = 0; + var totalSize = 0L; + + try + { + // Get all PNG files in cache directory + var pngFiles = Directory.GetFiles(_cacheDirectory, "*.png", SearchOption.TopDirectoryOnly); + + foreach (var file in pngFiles) + { + if (cancellationToken.IsCancellationRequested) + break; + + try + { + var fileInfo = new FileInfo(file); + var fileTime = fileInfo.LastWriteTimeUtc; + + // Delete if file is older than retention period + if (fileTime < cutoffTime) + { + var fileSize = fileInfo.Length; + File.Delete(file); + deletedCount++; + totalSize += fileSize; + + // Also delete associated metadata file if it exists + var metadataFile = FilePathHelper.GetMetadataFilePath(file); + if (File.Exists(metadataFile)) + { + File.Delete(metadataFile); + } + + _logger.LogDebug("Deleted old cache file: {File} (age: {Age})", + Path.GetFileName(file), DateTime.UtcNow - fileTime); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to delete cache file: {File}", file); + } + } + + if (deletedCount > 0) + { + var sizeInMB = totalSize / (1024.0 * 1024.0); + _logger.LogInformation("Cache cleanup completed. Deleted {Count} files ({Size:F2} MB)", + deletedCount, sizeInMB); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Error during cache file enumeration"); + } + + // Also clean up old debug directories + try + { + var debugDir = Path.Combine(_cacheDirectory, "debug"); + if (Directory.Exists(debugDir)) + { + var debugDirs = Directory.GetDirectories(debugDir); + foreach (var dir in debugDirs) + { + if (cancellationToken.IsCancellationRequested) + break; + + try + { + var dirInfo = new DirectoryInfo(dir); + if (dirInfo.LastWriteTimeUtc < cutoffTime) + { + Directory.Delete(dir, recursive: true); + _logger.LogDebug("Deleted old debug directory: {Dir}", dir); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to delete debug directory: {Dir}", dir); + } + } + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error cleaning up debug directories: {Error}", ex.Message); + } + + return Task.CompletedTask; + } +} + diff --git a/Services/CacheManagementService.cs b/Services/CacheManagementService.cs new file mode 100644 index 0000000..d00800e --- /dev/null +++ b/Services/CacheManagementService.cs @@ -0,0 +1,210 @@ +using BomLocalService.Services.Interfaces; +using BomLocalService.Utilities; +using Microsoft.Extensions.Hosting; + +namespace BomLocalService.Services; + +public class CacheManagementService : BackgroundService +{ + private readonly ILogger _logger; + private readonly IBomRadarService _bomRadarService; + private readonly IBrowserService _browserService; + private readonly IConfiguration _configuration; + private readonly TimeSpan _checkInterval; + private readonly HashSet _activeUpdates = new(); + private readonly object _lock = new(); + + public CacheManagementService( + ILogger logger, + IBomRadarService bomRadarService, + IBrowserService browserService, + IConfiguration configuration) + { + _logger = logger; + _bomRadarService = bomRadarService; + _browserService = browserService; + _configuration = configuration; + var checkIntervalMinutes = configuration.GetValue("CacheManagement:CheckIntervalMinutes", 5); + _checkInterval = TimeSpan.FromMinutes(checkIntervalMinutes); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _logger.LogInformation("Cache management service started. Check interval: {Interval}", _checkInterval); + + // Wait a bit for the service to fully initialize + var initialDelaySeconds = _configuration.GetValue("CacheManagement:InitialDelaySeconds", 10); + await Task.Delay(TimeSpan.FromSeconds(initialDelaySeconds), stoppingToken); + + // Pre-warm the browser before starting cache updates + _logger.LogInformation("Pre-warming browser before cache updates"); + try + { + await _browserService.InitializeAsync(stoppingToken); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to pre-warm browser, will continue anyway"); + } + + // Get initial list of locations to manage (could be from config or discovered from cache) + var locationsToManage = GetLocationsToManage(); + + // Initial cache update on startup + _logger.LogInformation("Performing initial cache update for {Count} locations", locationsToManage.Count); + int initialUpdatesTriggered = 0; + int initialCachesValid = 0; + + foreach (var (suburb, state) in locationsToManage) + { + if (stoppingToken.IsCancellationRequested) + break; + + var (triggered, isValid) = await UpdateCacheIfNeededAsync(suburb, state, stoppingToken); + if (triggered) + { + initialUpdatesTriggered++; + } + else if (isValid) + { + initialCachesValid++; + } + + // Stagger updates to avoid overwhelming the system + var updateStaggerSeconds = _configuration.GetValue("CacheManagement:UpdateStaggerSeconds", 2); + await Task.Delay(TimeSpan.FromSeconds(updateStaggerSeconds), stoppingToken); + } + + _logger.LogInformation("Initial cache update completed: {UpdatesTriggered} updates triggered, {CachesValid} caches valid", + initialUpdatesTriggered, initialCachesValid); + + // Periodic cache updates + while (!stoppingToken.IsCancellationRequested) + { + try + { + await Task.Delay(_checkInterval, stoppingToken); + + // Refresh list of locations (in case new ones were cached) + locationsToManage = GetLocationsToManage(); + _logger.LogInformation("Starting periodic cache check for {Count} locations", locationsToManage.Count); + + int updatesTriggered = 0; + int cachesValid = 0; + + foreach (var (suburb, state) in locationsToManage) + { + if (stoppingToken.IsCancellationRequested) + break; + + var (triggered, isValid) = await UpdateCacheIfNeededAsync(suburb, state, stoppingToken); + if (triggered) + { + updatesTriggered++; + } + else if (isValid) + { + cachesValid++; + } + + // Small delay between locations + var locationStaggerSeconds = _configuration.GetValue("CacheManagement:LocationStaggerSeconds", 1); + await Task.Delay(TimeSpan.FromSeconds(locationStaggerSeconds), stoppingToken); + } + + _logger.LogInformation("Periodic cache check completed: {UpdatesTriggered} updates triggered, {CachesValid} caches valid", + updatesTriggered, cachesValid); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error during cache management cycle"); + } + } + } + + private async Task<(bool updateTriggered, bool isValid)> UpdateCacheIfNeededAsync(string suburb, string state, CancellationToken cancellationToken) + { + var locationKey = LocationHelper.GetLocationKey(suburb, state); + + // Check if update is already in progress + lock (_lock) + { + if (_activeUpdates.Contains(locationKey)) + { + _logger.LogInformation("Update already in progress for {Suburb}, {State}, skipping", suburb, state); + return (false, false); + } + _activeUpdates.Add(locationKey); + } + + try + { + var status = await _bomRadarService.TriggerCacheUpdateAsync(suburb, state, cancellationToken); + + if (status.UpdateTriggered) + { + _logger.LogInformation("Cache update triggered for {Suburb}, {State} (expired at {ExpiresAt})", + suburb, state, status.CacheExpiresAt); + return (true, false); + } + else + { + _logger.LogInformation("Cache is valid for {Suburb}, {State} (expires at {ExpiresAt})", + suburb, state, status.CacheExpiresAt); + return (false, true); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Error checking/updating cache for {Suburb}, {State}", suburb, state); + return (false, false); + } + finally + { + lock (_lock) + { + _activeUpdates.Remove(locationKey); + } + } + } + + private List<(string suburb, string state)> GetLocationsToManage() + { + var locations = new List<(string, string)>(); + + try + { + // Get locations from existing cache files + var cacheDirectory = FilePathHelper.GetCacheDirectory(_configuration); + if (Directory.Exists(cacheDirectory)) + { + var pngFiles = Directory.GetFiles(cacheDirectory, "*.png", SearchOption.TopDirectoryOnly); + var seen = new HashSet(); + + foreach (var file in pngFiles) + { + var location = LocationHelper.ParseLocationFromFilename(file); + if (location.HasValue) + { + var locationKey = LocationHelper.GetLocationKey(location.Value.suburb, location.Value.state); + if (!seen.Contains(locationKey)) + { + seen.Add(locationKey); + locations.Add(location.Value); + } + } + } + } + + // Could also read from config if you want to pre-configure locations + // var configuredLocations = _configuration.GetSection("ManagedLocations").Get>(); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error discovering locations from cache"); + } + + return locations; + } +} + diff --git a/Services/CacheService.cs b/Services/CacheService.cs new file mode 100644 index 0000000..a4cc9ac --- /dev/null +++ b/Services/CacheService.cs @@ -0,0 +1,189 @@ +using BomLocalService.Models; +using BomLocalService.Services.Interfaces; +using BomLocalService.Utilities; +using System.Text.Json; + +namespace BomLocalService.Services; + +public class CacheService : ICacheService +{ + private readonly ILogger _logger; + private readonly string _cacheDirectory; + private readonly double _cacheExpirationMinutes; + + public CacheService(ILogger logger, IConfiguration configuration) + { + _logger = logger; + _cacheDirectory = FilePathHelper.GetCacheDirectory(configuration); + _cacheExpirationMinutes = configuration.GetValue("CacheExpirationMinutes", 15.5); + + // Ensure cache directory exists + Directory.CreateDirectory(_cacheDirectory); + } + + /// + /// Gets the cached screenshot path and metadata for a location + /// + public async Task<(string? screenshotPath, LastUpdatedInfo? metadata)> GetCachedScreenshotWithMetadataAsync( + string suburb, + string state, + CancellationToken cancellationToken = default) + { + var pattern = FilePathHelper.GetCacheFilePattern(suburb, state); + _logger.LogDebug("Looking for cached files matching pattern: {Pattern} in directory: {Directory}", pattern, _cacheDirectory); + + var files = Directory.GetFiles(_cacheDirectory, pattern) + .OrderByDescending(f => + { + // Try to extract timestamp from filename first + var timestamp = LocationHelper.ParseTimestampFromFilename(f); + if (timestamp.HasValue) + { + return timestamp.Value; + } + // Fallback to file write time + return File.GetLastWriteTime(f); + }) + .ToList(); + + var screenshotPath = files.FirstOrDefault(); + if (string.IsNullOrEmpty(screenshotPath) || !File.Exists(screenshotPath)) + { + return (null, null); + } + + var metadata = await LoadMetadataAsync(screenshotPath, cancellationToken); + return (screenshotPath, metadata); + } + + /// + /// Saves metadata alongside a screenshot + /// + public async Task SaveMetadataAsync(string screenshotPath, LastUpdatedInfo metadata, CancellationToken cancellationToken = default) + { + try + { + var metadataPath = FilePathHelper.GetMetadataFilePath(screenshotPath); + var json = JsonSerializer.Serialize(metadata, new JsonSerializerOptions { WriteIndented = true }); + // Use CancellationToken.None for file writes to avoid cancellation issues + // The screenshot is already saved, metadata save is best-effort + await File.WriteAllTextAsync(metadataPath, json, CancellationToken.None); + _logger.LogDebug("Saved metadata to: {Path}", metadataPath); + } + catch (Exception ex) + { + // Log but don't fail - screenshot is already saved + _logger.LogWarning(ex, "Failed to save metadata for screenshot: {Path}", screenshotPath); + } + } + + /// + /// Loads metadata for a screenshot + /// + public async Task LoadMetadataAsync(string screenshotPath, CancellationToken cancellationToken = default) + { + var metadataPath = FilePathHelper.GetMetadataFilePath(screenshotPath); + if (!File.Exists(metadataPath)) + { + return null; + } + + try + { + var json = await File.ReadAllTextAsync(metadataPath, cancellationToken); + return JsonSerializer.Deserialize(json); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to load metadata from: {Path}", metadataPath); + return null; + } + } + + /// + /// Checks if cache is still valid based on observation time + /// + public bool IsCacheValid(LastUpdatedInfo? metadata) + { + if (metadata == null) + { + return false; + } + + // BOM updates observations at :00, :15, :30, :45 (every 15 minutes by default) + // Cache expiration includes buffer (e.g., 15.5 minutes = 15 min + 30 sec buffer) + var nextUpdateTime = metadata.ObservationTime.AddMinutes(_cacheExpirationMinutes); + return DateTime.UtcNow < nextUpdateTime; + } + + /// + /// Gets the path to the cached screenshot for a location (simple version, ordered by creation time) + /// + public Task GetCachedScreenshotPathAsync(string suburb, string state, CancellationToken cancellationToken = default) + { + var pattern = FilePathHelper.GetCacheFilePattern(suburb, state); + var files = Directory.GetFiles(_cacheDirectory, pattern) + .OrderByDescending(f => File.GetCreationTime(f)) + .ToList(); + + return Task.FromResult(files.FirstOrDefault() ?? string.Empty); + } + + /// + /// Deletes all cached files for a location + /// + public Task DeleteCachedLocationAsync(string suburb, string state, CancellationToken cancellationToken = default) + { + var pattern = FilePathHelper.GetCacheFilePattern(suburb, state); + var deleted = false; + + try + { + // Delete all PNG files for this location + var pngFiles = Directory.GetFiles(_cacheDirectory, pattern); + foreach (var pngFile in pngFiles) + { + try + { + File.Delete(pngFile); + _logger.LogInformation("Deleted cached screenshot: {File}", pngFile); + deleted = true; + + // Also delete associated metadata JSON file + var metadataFile = FilePathHelper.GetMetadataFilePath(pngFile); + if (File.Exists(metadataFile)) + { + File.Delete(metadataFile); + _logger.LogInformation("Deleted metadata file: {File}", metadataFile); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to delete file: {File}", pngFile); + } + } + + if (deleted) + { + _logger.LogInformation("Deleted all cached files for location: {Suburb}, {State}", suburb, state); + } + else + { + _logger.LogDebug("No cached files found to delete for location: {Suburb}, {State}", suburb, state); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Error deleting cached location: {Suburb}, {State}", suburb, state); + return Task.FromResult(false); + } + + return Task.FromResult(deleted); + } + + /// + /// Gets the cache directory path + /// + public string GetCacheDirectory() => _cacheDirectory; +} + diff --git a/Services/DebugService.cs b/Services/DebugService.cs new file mode 100644 index 0000000..f3af430 --- /dev/null +++ b/Services/DebugService.cs @@ -0,0 +1,176 @@ +using BomLocalService.Services.Interfaces; +using BomLocalService.Utilities; +using Microsoft.Playwright; + +namespace BomLocalService.Services; + + public class DebugService : IDebugService + { + private readonly ILogger _logger; + private readonly bool _enabled; + private readonly string _debugDirectory; + private readonly int _waitMs; + + public DebugService(ILogger logger, IConfiguration configuration) + { + _logger = logger; + _waitMs = configuration.GetValue("Debug:WaitMs", 2000); + + // Check all possible ways the config might be set + var debugEnabledEnv = Environment.GetEnvironmentVariable("DEBUG__ENABLED"); + var debugEnabledConfig = configuration.GetValue("Debug:Enabled", false); + var debugEnabledFromConfig = configuration["Debug:Enabled"]; + + _logger.LogInformation("Debug configuration check - Env var DEBUG__ENABLED: {EnvVar}, Config Debug:Enabled: {ConfigValue}, Config string: {ConfigString}", + debugEnabledEnv ?? "null", debugEnabledConfig, debugEnabledFromConfig ?? "null"); + + // Try environment variable first, then config + _enabled = !string.IsNullOrEmpty(debugEnabledEnv) + ? bool.TryParse(debugEnabledEnv, out var envBool) && envBool + : debugEnabledConfig; + + var cacheDirectory = FilePathHelper.GetCacheDirectory(configuration); + _debugDirectory = Path.Combine(cacheDirectory, "debug"); + + if (_enabled) + { + Directory.CreateDirectory(_debugDirectory); + _logger.LogInformation("Debug mode is ENABLED. Debug files will be saved to: {DebugDirectory}", _debugDirectory); + } + else + { + _logger.LogInformation("Debug mode is DISABLED"); + } + } + + public bool IsEnabled => _enabled; + + public string CreateRequestFolder(string requestId) + { + if (!_enabled) return string.Empty; + + var requestFolder = Path.Combine(_debugDirectory, requestId); + Directory.CreateDirectory(requestFolder); + _logger.LogDebug("Created debug folder for request: {RequestId} at {Path}", requestId, requestFolder); + return requestFolder; + } + + public async Task SaveStepDebugAsync(string requestFolder, int stepNumber, string stepName, IPage page, List<(string type, string text, DateTime timestamp)>? consoleMessages = null, List<(string method, string url, int? status, string resourceType, DateTime timestamp)>? networkRequests = null, CancellationToken cancellationToken = default) + { + if (!_enabled || string.IsNullOrEmpty(requestFolder)) return; + + try + { + // Wait for page to be fully loaded before taking screenshot + try + { + await page.WaitForLoadStateAsync(LoadState.NetworkIdle, new PageWaitForLoadStateOptions { Timeout = 10000 }); + await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 5000 }); + } + catch + { + // If network idle times out, at least wait for DOM + } + await page.WaitForTimeoutAsync(_waitMs); // Additional wait for dynamic content and animations + + var stepFolder = Path.Combine(requestFolder, $"step_{stepNumber:D2}_{stepName}"); + Directory.CreateDirectory(stepFolder); + + // Save screenshot + var screenshotPath = Path.Combine(stepFolder, "screenshot.png"); + await page.ScreenshotAsync(new PageScreenshotOptions + { + Path = screenshotPath, + FullPage = true + }); + + // Save HTML + var htmlContent = await page.ContentAsync(); + var htmlPath = Path.Combine(stepFolder, "page.html"); + await File.WriteAllTextAsync(htmlPath, htmlContent, cancellationToken); + + // Save console messages if provided + if (consoleMessages != null && consoleMessages.Count > 0) + { + var consolePath = Path.Combine(stepFolder, "console.log"); + var consoleText = string.Join("\n", consoleMessages.Select(msg => $"[{msg.timestamp:HH:mm:ss.fff}] [{msg.type}] {msg.text}")); + await File.WriteAllTextAsync(consolePath, consoleText, cancellationToken); + } + + // Save network requests summary if provided + if (networkRequests != null && networkRequests.Count > 0) + { + var networkPath = Path.Combine(stepFolder, "network.log"); + var networkText = string.Join("\n", networkRequests.Select(req => + $"[{req.timestamp:HH:mm:ss.fff}] {req.method} {req.url} -> {req.status?.ToString() ?? "pending"} ({req.resourceType})" + )); + await File.WriteAllTextAsync(networkPath, networkText, cancellationToken); + } + + _logger.LogDebug("Saved debug files for step {StepNumber} ({StepName}) to {Path}", stepNumber, stepName, stepFolder); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to save debug files for step {StepNumber} ({StepName})", stepNumber, stepName); + } + } + + public async Task SaveErrorDebugAsync(string requestFolder, string errorMessage, IPage? page = null, + List<(string type, string text, DateTime timestamp)>? consoleMessages = null, + List<(string method, string url, int? status, string resourceType, DateTime timestamp)>? networkRequests = null, + CancellationToken cancellationToken = default) + { + if (!_enabled || string.IsNullOrEmpty(requestFolder)) return; + + try + { + var errorFolder = Path.Combine(requestFolder, "error"); + Directory.CreateDirectory(errorFolder); + + // Save error message + var errorPath = Path.Combine(errorFolder, "error.txt"); + await File.WriteAllTextAsync(errorPath, errorMessage, cancellationToken); + + // Save screenshot if page is available + if (page != null) + { + var screenshotPath = Path.Combine(errorFolder, "screenshot.png"); + await page.ScreenshotAsync(new PageScreenshotOptions + { + Path = screenshotPath, + FullPage = true + }); + + // Save HTML + var htmlContent = await page.ContentAsync(); + var htmlPath = Path.Combine(errorFolder, "page.html"); + await File.WriteAllTextAsync(htmlPath, htmlContent, cancellationToken); + } + + // Save console messages if provided + if (consoleMessages != null && consoleMessages.Count > 0) + { + var consolePath = Path.Combine(errorFolder, "console.log"); + var consoleText = string.Join("\n", consoleMessages.Select(msg => $"[{msg.timestamp:HH:mm:ss.fff}] [{msg.type}] {msg.text}")); + await File.WriteAllTextAsync(consolePath, consoleText, cancellationToken); + } + + // Save network requests summary if provided + if (networkRequests != null && networkRequests.Count > 0) + { + var networkPath = Path.Combine(errorFolder, "network.log"); + var networkText = string.Join("\n", networkRequests.Select(req => + $"[{req.timestamp:HH:mm:ss.fff}] {req.method} {req.url} -> {req.status?.ToString() ?? "pending"} ({req.resourceType})" + )); + await File.WriteAllTextAsync(networkPath, networkText, cancellationToken); + } + + _logger.LogDebug("Saved error debug files to {Path}", errorFolder); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to save error debug files"); + } + } +} + diff --git a/Services/Interfaces/IBomRadarService.cs b/Services/Interfaces/IBomRadarService.cs new file mode 100644 index 0000000..8724a73 --- /dev/null +++ b/Services/Interfaces/IBomRadarService.cs @@ -0,0 +1,60 @@ +using BomLocalService.Models; + +namespace BomLocalService.Services.Interfaces; + +/// +/// Main service interface for BOM radar screenshot operations. +/// Orchestrates cache management, browser automation, and web scraping to provide radar screenshots for Australian locations. +/// +public interface IBomRadarService +{ + /// + /// Gets a cached radar screenshot for a location. + /// Returns the screenshot response if available in cache, otherwise returns null. + /// + /// The suburb name (e.g., "Pomona", "Brisbane") + /// The Australian state abbreviation (e.g., "QLD", "NSW", "VIC") + /// Cancellation token to cancel the operation + /// Radar screenshot response with image path and metadata, or null if not cached + Task GetCachedScreenshotAsync(string suburb, string state, CancellationToken cancellationToken = default); + + /// + /// Triggers a cache update for a location. + /// If cache is missing or expired, initiates a background update and returns status information. + /// + /// The suburb name (e.g., "Pomona", "Brisbane") + /// The Australian state abbreviation (e.g., "QLD", "NSW", "VIC") + /// Cancellation token to cancel the operation + /// Status information about the cache update operation + Task TriggerCacheUpdateAsync(string suburb, string state, CancellationToken cancellationToken = default); + + /// + /// Gets metadata about the cached radar data for a location. + /// Returns observation time, forecast time, weather station, and distance information. + /// + /// The suburb name (e.g., "Pomona", "Brisbane") + /// The Australian state abbreviation (e.g., "QLD", "NSW", "VIC") + /// Cancellation token to cancel the operation + /// Last updated information, or null if no cached data exists + Task GetLastUpdatedInfoAsync(string suburb, string state, CancellationToken cancellationToken = default); + + /// + /// Gets the file system path to the cached screenshot for a location. + /// Returns empty string if no cached screenshot exists. + /// + /// The suburb name (e.g., "Pomona", "Brisbane") + /// The Australian state abbreviation (e.g., "QLD", "NSW", "VIC") + /// Cancellation token to cancel the operation + /// File system path to the cached screenshot, or empty string if not found + Task GetCachedScreenshotPathAsync(string suburb, string state, CancellationToken cancellationToken = default); + + /// + /// Deletes all cached files (screenshot and metadata) for a location. + /// + /// The suburb name (e.g., "Pomona", "Brisbane") + /// The Australian state abbreviation (e.g., "QLD", "NSW", "VIC") + /// Cancellation token to cancel the operation + /// True if files were deleted, false if no cached files existed + Task DeleteCachedLocationAsync(string suburb, string state, CancellationToken cancellationToken = default); +} + diff --git a/Services/Interfaces/IBrowserService.cs b/Services/Interfaces/IBrowserService.cs new file mode 100644 index 0000000..2b0d4f6 --- /dev/null +++ b/Services/Interfaces/IBrowserService.cs @@ -0,0 +1,44 @@ +using Microsoft.Playwright; + +namespace BomLocalService.Services.Interfaces; + +/// +/// Service interface for managing Playwright browser instances and automation. +/// Handles browser lifecycle, context creation, and anti-detection measures for web scraping. +/// +public interface IBrowserService : IDisposable +{ + /// + /// Creates a new browser context with proper configuration for BOM website scraping. + /// Configures viewport, user agent, timezone, and HTTP headers to mimic a real browser. + /// + /// A configured browser context ready for page creation + Task CreateContextAsync(); + + /// + /// Creates a new page with anti-detection scripts and optional debug event handlers. + /// Injects comprehensive anti-detection JavaScript to avoid bot detection. + /// If debug mode is enabled, captures console messages and network requests for debugging. + /// + /// The browser context to create the page in + /// Unique request identifier for debug file organization + /// Tuple containing the page, list of console messages, and list of network requests + Task<(IPage page, List<(string type, string text, DateTime timestamp)> consoleMessages, List<(string method, string url, int? status, string resourceType, DateTime timestamp)> networkRequests)> CreatePageWithDebugAsync( + IBrowserContext context, + string requestId); + + /// + /// Initializes and pre-warms the browser instance. + /// Launches the Chromium browser to reduce latency on first use. + /// + /// Cancellation token to cancel the operation + Task InitializeAsync(CancellationToken cancellationToken = default); + + /// + /// Gets the semaphore used to synchronize browser operations. + /// Ensures only one browser operation runs at a time to prevent resource conflicts. + /// + /// The semaphore for synchronizing browser access + SemaphoreSlim GetSemaphore(); +} + diff --git a/Services/Interfaces/ICacheService.cs b/Services/Interfaces/ICacheService.cs new file mode 100644 index 0000000..ea17e37 --- /dev/null +++ b/Services/Interfaces/ICacheService.cs @@ -0,0 +1,67 @@ +using BomLocalService.Models; + +namespace BomLocalService.Services.Interfaces; + +/// +/// Service interface for managing cached radar screenshot files and metadata. +/// Handles file system operations for storing and retrieving cached BOM radar screenshots. +/// +public interface ICacheService +{ + /// + /// Gets the cached screenshot file path and associated metadata for a location. + /// Searches for PNG files matching the location pattern and loads the corresponding metadata JSON file. + /// + /// The suburb name (e.g., "Pomona", "Brisbane") + /// The Australian state abbreviation (e.g., "QLD", "NSW", "VIC") + /// Cancellation token to cancel the operation + /// Tuple containing the screenshot file path and metadata, or (null, null) if not found + Task<(string? screenshotPath, LastUpdatedInfo? metadata)> GetCachedScreenshotWithMetadataAsync( + string suburb, + string state, + CancellationToken cancellationToken = default); + + /// + /// Saves metadata JSON file alongside a screenshot file. + /// Creates a JSON file with the same base name as the screenshot but with .json extension. + /// + /// Full path to the screenshot PNG file + /// Metadata to save (observation time, forecast time, weather station, distance) + /// Cancellation token to cancel the operation + Task SaveMetadataAsync(string screenshotPath, LastUpdatedInfo metadata, CancellationToken cancellationToken = default); + + /// + /// Checks if cached metadata is still valid (not expired). + /// Cache is valid if observation time + expiration buffer (typically 15.5 minutes) is still in the future. + /// BOM updates observations every 15 minutes, so the buffer accounts for timing variations. + /// + /// The metadata to validate, or null + /// True if metadata exists and cache is still valid, false otherwise + bool IsCacheValid(LastUpdatedInfo? metadata); + + /// + /// Gets the file system path to the most recent cached screenshot for a location. + /// Returns files ordered by creation time, most recent first. + /// + /// The suburb name (e.g., "Pomona", "Brisbane") + /// The Australian state abbreviation (e.g., "QLD", "NSW", "VIC") + /// Cancellation token to cancel the operation + /// File system path to the cached screenshot, or empty string if not found + Task GetCachedScreenshotPathAsync(string suburb, string state, CancellationToken cancellationToken = default); + + /// + /// Deletes all cached files (PNG screenshots and JSON metadata) for a location. + /// + /// The suburb name (e.g., "Pomona", "Brisbane") + /// The Australian state abbreviation (e.g., "QLD", "NSW", "VIC") + /// Cancellation token to cancel the operation + /// True if any files were deleted, false if no cached files existed + Task DeleteCachedLocationAsync(string suburb, string state, CancellationToken cancellationToken = default); + + /// + /// Gets the cache directory path where screenshots and metadata are stored. + /// + /// The full path to the cache directory + string GetCacheDirectory(); +} + diff --git a/Services/Interfaces/IDebugService.cs b/Services/Interfaces/IDebugService.cs new file mode 100644 index 0000000..77b889f --- /dev/null +++ b/Services/Interfaces/IDebugService.cs @@ -0,0 +1,72 @@ +using Microsoft.Playwright; + +namespace BomLocalService.Services.Interfaces; + +/// +/// Service interface for debug file generation during web scraping operations. +/// When enabled, saves screenshots, HTML, console logs, and network request information for troubleshooting. +/// +public interface IDebugService +{ + /// + /// Indicates whether debug mode is enabled. + /// When false, debug methods return immediately without saving files. + /// + bool IsEnabled { get; } + + /// + /// Creates a folder for storing debug files for a specific request. + /// Folder structure: {CacheDirectory}/debug/{requestId}/ + /// + /// Unique identifier for the request (e.g., timestamp + GUID) + /// Full path to the created debug folder, or empty string if debug is disabled + string CreateRequestFolder(string requestId); + + /// + /// Saves debug files for a scraping step. + /// Creates a step folder (e.g., "step_01_homepage_loaded") containing: + /// - screenshot.png (full page screenshot) + /// - page.html (complete HTML source) + /// - console.log (browser console messages) + /// - network.log (network request summary) + /// + /// The debug folder path for this request (from CreateRequestFolder) + /// Step number for ordering (e.g., 1, 2, 3) + /// Descriptive name for the step (e.g., "homepage_loaded", "search_button_clicked") + /// The Playwright page instance to capture + /// Optional list of console messages to save + /// Optional list of network requests to save + /// Cancellation token to cancel the operation + Task SaveStepDebugAsync( + string requestFolder, + int stepNumber, + string stepName, + IPage page, + List<(string type, string text, DateTime timestamp)>? consoleMessages = null, + List<(string method, string url, int? status, string resourceType, DateTime timestamp)>? networkRequests = null, + CancellationToken cancellationToken = default); + + /// + /// Saves debug files when an error occurs during scraping. + /// Creates an "error" subfolder containing: + /// - error.txt (error message and stack trace) + /// - screenshot.png (page state at error time, if page available) + /// - page.html (HTML source at error time, if page available) + /// - console.log (console messages up to error) + /// - network.log (network requests up to error) + /// + /// The debug folder path for this request (from CreateRequestFolder) + /// The error message and/or stack trace to save + /// Optional Playwright page instance to capture error state + /// Optional list of console messages to save + /// Optional list of network requests to save + /// Cancellation token to cancel the operation + Task SaveErrorDebugAsync( + string requestFolder, + string errorMessage, + IPage? page = null, + List<(string type, string text, DateTime timestamp)>? consoleMessages = null, + List<(string method, string url, int? status, string resourceType, DateTime timestamp)>? networkRequests = null, + CancellationToken cancellationToken = default); +} + diff --git a/Services/Interfaces/IScrapingService.cs b/Services/Interfaces/IScrapingService.cs new file mode 100644 index 0000000..a6d1316 --- /dev/null +++ b/Services/Interfaces/IScrapingService.cs @@ -0,0 +1,34 @@ +using BomLocalService.Models; +using Microsoft.Playwright; + +namespace BomLocalService.Services.Interfaces; + +/// +/// Service interface for scraping radar screenshots from the BOM website. +/// Orchestrates the multi-step process of navigating BOM, finding locations, and capturing radar screenshots. +/// +public interface IScrapingService +{ + /// + /// Scrapes a radar screenshot from the BOM website for a given location. + /// Performs the complete workflow: navigates to BOM, searches for location, navigates to radar page, + /// waits for map to render, captures screenshot, extracts metadata, and saves to cache. + /// + /// The suburb name (e.g., "Pomona", "Brisbane") + /// The Australian state abbreviation (e.g., "QLD", "NSW", "VIC") + /// Folder path for saving debug files (screenshots, HTML, logs) if debug mode is enabled + /// The Playwright page instance to use for scraping + /// List to capture console messages from the browser (for debugging) + /// List to capture network request information (for debugging) + /// Cancellation token to cancel the operation + /// Radar screenshot response with image path and metadata + Task ScrapeRadarScreenshotAsync( + string suburb, + string state, + string debugFolder, + IPage page, + List<(string type, string text, DateTime timestamp)> consoleMessages, + List<(string method, string url, int? status, string resourceType, DateTime timestamp)> networkRequests, + CancellationToken cancellationToken = default); +} + diff --git a/Services/Interfaces/ITimeParsingService.cs b/Services/Interfaces/ITimeParsingService.cs new file mode 100644 index 0000000..11ae08b --- /dev/null +++ b/Services/Interfaces/ITimeParsingService.cs @@ -0,0 +1,29 @@ +using BomLocalService.Models; +using Microsoft.Playwright; + +namespace BomLocalService.Services.Interfaces; + +/// +/// Service interface for parsing time and metadata information from BOM website content. +/// Extracts observation times, forecast times, weather station names, and distances from BOM weather map pages. +/// +public interface ITimeParsingService +{ + /// + /// Extracts last updated information from a BOM weather map page. + /// Scrapes the weather metadata section to get observation time, forecast time, weather station, and distance. + /// + /// The Playwright page instance containing the loaded BOM weather map + /// Last updated information with parsed times and metadata + Task ExtractLastUpdatedInfoAsync(IPage page); + + /// + /// Parses last updated text from BOM website to extract structured information. + /// Handles various text formats like "Observations: 11 minutes ago, 8:20 pm AEST at Gympie weather station, 30 km from Pomona, QLD". + /// Converts times to UTC using the configured timezone. + /// + /// The raw text content from the BOM weather metadata section + /// Parsed last updated information with observation time, forecast time, weather station, and distance + LastUpdatedInfo ParseLastUpdatedText(string text); +} + diff --git a/Services/ScrapingService.cs b/Services/ScrapingService.cs new file mode 100644 index 0000000..6bdef20 --- /dev/null +++ b/Services/ScrapingService.cs @@ -0,0 +1,373 @@ +using BomLocalService.Models; +using BomLocalService.Services.Interfaces; +using BomLocalService.Utilities; +using Microsoft.Playwright; + +namespace BomLocalService.Services; + +public class ScrapingService : IScrapingService +{ + private readonly ILogger _logger; + private readonly IBrowserService _browserService; + private readonly ITimeParsingService _timeParsingService; + private readonly ICacheService _cacheService; + private readonly IDebugService _debugService; + private readonly int _dynamicContentWaitMs; + private readonly int _tileRenderWaitMs; + + // Selector constants + private static readonly string[] SearchButtonSelectors = new[] + { + "button[data-testid='searchLabel']", + "button[aria-label='Search for a location']", + "button.search-location__trigger-button", + "button.bom-button:has(span.bom-button__label:has-text('Search for a location'))", + "button:has-text('Search for a location')", + "[data-testid='showSearchModal'] button" + }; + + private static readonly string[] RadarLinkSelectors = new[] + { + "a.jump-link.bom-button.bom-button--secondary:has-text('Rain radar and weather map')", + ".cta.button a.jump-link:has-text('Rain radar and weather map')", + "a.jump-link:has-text('Rain radar and weather map')", + ".cta.button a:has-text('Rain radar and weather map')", + "a.bom-button--secondary:has-text('Rain radar and weather map')", + "a:has-text('Rain radar and weather map')", + "a:has-text('rain radar')", + "a[href*='radar']" + }; + + public ScrapingService( + ILogger logger, + IBrowserService browserService, + ITimeParsingService timeParsingService, + ICacheService cacheService, + IDebugService debugService, + IConfiguration configuration) + { + _logger = logger; + _browserService = browserService; + _timeParsingService = timeParsingService; + _cacheService = cacheService; + _debugService = debugService; + _dynamicContentWaitMs = configuration.GetValue("Screenshot:DynamicContentWaitMs", 2000); + _tileRenderWaitMs = configuration.GetValue("Screenshot:TileRenderWaitMs", 5000); + } + + /// + /// Scrapes the BOM website to get a radar screenshot for a location + /// + public async Task ScrapeRadarScreenshotAsync( + string suburb, + string state, + string debugFolder, + IPage page, + List<(string type, string text, DateTime timestamp)> consoleMessages, + List<(string method, string url, int? status, string resourceType, DateTime timestamp)> networkRequests, + CancellationToken cancellationToken = default) + { + try + { + // Step 1: Navigate to BOM homepage and wait for search button + _logger.LogInformation("Navigating to BOM homepage"); + await page.GotoAsync("https://www.bom.gov.au/", new PageGotoOptions + { + WaitUntil = WaitUntilState.DOMContentLoaded, + Timeout = 30000 + }); + + // Wait for search button to be ready instead of NetworkIdle (faster) + var searchButtonReady = page.Locator("button[data-testid='searchLabel'], button[aria-label='Search for a location'], button.search-location__trigger-button").First; + await searchButtonReady.WaitForAsync(new LocatorWaitForOptions { Timeout = 10000, State = WaitForSelectorState.Visible }); + await _debugService.SaveStepDebugAsync(debugFolder, 1, "homepage_loaded", page, consoleMessages, networkRequests, cancellationToken); + + // Step 2: Click "Search for a location" button to open search UI + _logger.LogInformation("Clicking 'Search for a location' button"); + var searchButton = await SelectorHelper.FindLocatorBySelectorsAsync(page, SearchButtonSelectors, _logger); + + if (searchButton == null) + { + var errorMsg = "Could not find 'Search for a location' button on BOM homepage."; + await _debugService.SaveErrorDebugAsync(debugFolder, errorMsg, page, consoleMessages, networkRequests, cancellationToken); + throw new Exception(errorMsg); + } + + await searchButton.ClickAsync(); + + // Wait for search input to appear + var searchInputReady = page.Locator("#search-enter-keyword").First; + await searchInputReady.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000, State = WaitForSelectorState.Visible }); + await _debugService.SaveStepDebugAsync(debugFolder, 2, "search_button_clicked", page, consoleMessages, networkRequests, cancellationToken); + + // Step 3: Find and fill search box with suburb name + _logger.LogInformation("Searching for suburb: {Suburb}", suburb); + var searchInput = page.Locator("#search-enter-keyword").First; + await searchInput.FillAsync(suburb); + + // Step 4: Wait for autocomplete suggestions to appear + _logger.LogInformation("Waiting for autocomplete suggestions"); + await page.WaitForFunctionAsync(@"() => { + const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=""listitem""]')); + return results.length > 0 && results.some(r => r.offsetParent !== null); + }", new PageWaitForFunctionOptions { Timeout = 10000 }); + await _debugService.SaveStepDebugAsync(debugFolder, 3, "search_input_filled", page, consoleMessages, networkRequests, cancellationToken); + + // Step 5: Find the matching search result based on suburb and state + _logger.LogInformation("Looking for matching search result for {Suburb}, {State}", suburb, state); + var suburbLower = suburb.ToLower().Trim(); + var stateLower = state.ToLower().Trim(); + + // Get the actual count from the summary element + var summaryText = await page.Locator("#location-results-title, [data-testid='location-results-title']").First.TextContentAsync(); + int? actualCount = null; + if (!string.IsNullOrEmpty(summaryText)) + { + // Parse "3 of 3 location results" or similar + var countMatch = System.Text.RegularExpressions.Regex.Match(summaryText, @"(\d+)\s+of\s+(\d+)", System.Text.RegularExpressions.RegexOptions.IgnoreCase); + if (countMatch.Success && countMatch.Groups.Count >= 3) + { + if (int.TryParse(countMatch.Groups[2].Value, out var total)) + { + actualCount = total; + } + } + } + + // Fetch all result data in one JavaScript evaluation - extract location name and state separately + List<(string name, string desc, string fullText)> results = new(); + try + { + var resultData = await page.EvaluateAsync(@"() => { + // Scope to the location results list - find the ul with aria-labelledby pointing to location-results-title + const resultsList = document.querySelector('ul[aria-labelledby=""location-results-title""]'); + if (!resultsList) { + console.log('Location results list not found'); + return []; + } + const results = Array.from(resultsList.querySelectorAll('li.bom-linklist__item[role=""listitem""]')); + console.log('Found', results.length, 'location results'); + return results.map((r, index) => { + // Query from the li element - elements are nested inside tag + const nameEl = r.querySelector('[data-testid=""location-name""]'); + const descEl = r.querySelector('.bom-linklist-item__desc'); + const name = nameEl ? (nameEl.textContent || nameEl.innerText || '').trim() : ''; + const desc = descEl ? (descEl.textContent || descEl.innerText || '').trim() : ''; + const fullText = (r.textContent || r.innerText || '').trim(); + console.log('Result', index, ':', { hasNameEl: !!nameEl, hasDescEl: !!descEl, name: name, desc: desc }); + return [name, desc, fullText]; + }); + }"); + + // Convert to structured data + results = resultData.Select(arr => ( + name: arr.Length > 0 ? arr[0] : "", + desc: arr.Length > 1 ? arr[1] : "", + fullText: arr.Length > 2 ? arr[2] : "" + )).ToList(); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to extract structured result data, falling back to text content"); + // Fallback to simple text extraction + var resultTexts = await page.EvaluateAsync(@"() => { + const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=""listitem""]')); + return results.map(r => r.textContent || ''); + }"); + results = resultTexts.Select(text => (name: "", desc: "", fullText: text)).ToList(); + } + + if (actualCount.HasValue) + { + _logger.LogInformation("Found {Count} location results (summary: {Summary})", actualCount.Value, summaryText?.Trim()); + // Only use the first N results that match the actual count + if (results.Count > actualCount.Value) + { + results = results.Take(actualCount.Value).ToList(); + } + } + else + { + _logger.LogInformation("Found {Count} search results", results.Count); + } + + int? matchingIndex = null; + for (int i = 0; i < results.Count; i++) + { + var (name, desc, fullText) = results[i]; + var nameLower = name.ToLower().Trim(); + var descLower = desc.ToLower().Trim(); + var fullTextLower = fullText.ToLower(); + + _logger.LogInformation("Checking result {Index}: Name='{Name}', Desc='{Desc}', FullText='{FullText}'", + i, name, desc, fullText.Length > 100 ? fullText.Substring(0, 100) + "..." : fullText); + + // Check if suburb name matches (from location-name element or fallback to fullText) + var matchesSuburb = false; + if (!string.IsNullOrEmpty(name)) + { + matchesSuburb = nameLower == suburbLower || nameLower.Contains(suburbLower); + } + else + { + // Fallback to fullText if name extraction failed + matchesSuburb = fullTextLower.Contains(suburbLower); + } + + // Check if state matches (from description which contains "Queensland 4300" or "New South Wales 2469") + var matchesState = false; + if (!string.IsNullOrEmpty(desc)) + { + matchesState = StateAbbreviationHelper.MatchesState(descLower, stateLower); + } + // Always check fullText for state as fallback + if (!matchesState) + { + matchesState = StateAbbreviationHelper.MatchesState(fullTextLower, stateLower); + } + + _logger.LogInformation("Result {Index}: matchesSuburb={MatchesSuburb} (suburb='{Suburb}' vs name='{Name}'), matchesState={MatchesState} (state='{State}' vs desc='{Desc}')", + i, matchesSuburb, suburbLower, nameLower, matchesState, stateLower, descLower); + + if (matchesSuburb && matchesState) + { + matchingIndex = i; + _logger.LogInformation("Found matching result: {Name} - {Desc}", name, desc); + break; + } + } + + // Click the matching result (or first if no match) + // Scope to the location results list + var resultsList = page.Locator("ul[aria-labelledby='location-results-title']"); + var allResults = resultsList.Locator("li.bom-linklist__item[role='listitem']"); + var resultToClick = matchingIndex.HasValue ? allResults.Nth(matchingIndex.Value) : allResults.First; + + if (!matchingIndex.HasValue) + { + _logger.LogInformation("No exact match found, using first result"); + } + + await resultToClick.ClickAsync(); + await _debugService.SaveStepDebugAsync(debugFolder, 4, "search_result_selected", page, consoleMessages, networkRequests, cancellationToken); + + // Wait for forecast page to load + await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15000 }); + await page.WaitForTimeoutAsync(_dynamicContentWaitMs); // Brief wait for dynamic content + await _debugService.SaveStepDebugAsync(debugFolder, 5, "forecast_page_loaded", page, consoleMessages, networkRequests, cancellationToken); + + // Step 6: Find and click "Rain radar and weather map" link + _logger.LogInformation("Looking for 'Rain radar and weather map' link"); + var radarLink = await SelectorHelper.FindLocatorBySelectorsAsync(page, RadarLinkSelectors, _logger); + + if (radarLink == null) + { + var errorMsg = $"Could not find 'Rain radar and weather map' link for {suburb}, {state}"; + await _debugService.SaveErrorDebugAsync(debugFolder, errorMsg, page, consoleMessages, networkRequests, cancellationToken); + throw new Exception(errorMsg); + } + + await radarLink.ClickAsync(); + await _debugService.SaveStepDebugAsync(debugFolder, 6, "radar_link_clicked", page, consoleMessages, networkRequests, cancellationToken); + + // Step 7: Wait for weather map page to load and map to fully render + _logger.LogInformation("Waiting for weather map page to load"); + await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15000 }); + + // Wait for the map canvas element to appear with proper dimensions + _logger.LogInformation("Waiting for map canvas element to render"); + var mapCanvas = page.Locator(".esri-view-surface canvas").First; + await mapCanvas.WaitForAsync(new LocatorWaitForOptions { Timeout = 15000 }); + + // Wait for canvas to have valid dimensions + await page.WaitForFunctionAsync(@"() => { + const canvas = document.querySelector('.esri-view-surface canvas'); + return canvas && canvas.width > 0 && canvas.height > 0 && canvas.offsetWidth > 0 && canvas.offsetHeight > 0; + }", new PageWaitForFunctionOptions { Timeout = 15000 }); + + _logger.LogInformation("Map canvas is ready - waiting for map to render"); + + // Wait for Esri map view to be ready + try + { + await page.WaitForFunctionAsync(@"() => { + try { + const elements = document.querySelectorAll('.esri-view'); + for (let el of elements) { + if (el.__view && el.__view.ready) { + return true; + } + } + } catch(e) {} + return false; + }", new PageWaitForFunctionOptions { Timeout = 30000 }); + _logger.LogInformation("Esri map view is ready"); + } + catch + { + _logger.LogInformation("Esri view ready check timed out, continuing with fixed wait"); + } + + // Additional wait for tiles to render + await page.WaitForTimeoutAsync(_tileRenderWaitMs); + + await _debugService.SaveStepDebugAsync(debugFolder, 7, "weather_map_ready", page, consoleMessages, networkRequests, cancellationToken); + + // Step 8: Extract last updated information + var lastUpdatedInfo = await _timeParsingService.ExtractLastUpdatedInfoAsync(page); + + // Step 9: Find the map container and take screenshot of just the map area + _logger.LogInformation("Taking screenshot of map area"); + var mapContainer = page.Locator(".esri-view-surface").First; + await mapContainer.WaitForAsync(new LocatorWaitForOptions { Timeout = 10000 }); + + // Ensure the map container is visible and has dimensions before taking screenshot + await page.WaitForFunctionAsync(@"() => { + const container = document.querySelector('.esri-view-surface'); + return container && container.offsetWidth > 0 && container.offsetHeight > 0; + }", new PageWaitForFunctionOptions { Timeout = 10000 }); + + // Get bounding box of map container + var boundingBox = await mapContainer.BoundingBoxAsync(); + if (boundingBox == null) + { + throw new Exception("Could not determine map container bounds"); + } + + // Generate cache filename based on suburb, state and timestamp + var locationKey = LocationHelper.GetLocationKey(suburb, state); + var safeLocationKey = LocationHelper.SanitizeFileName(locationKey); + var timestamp = DateTime.UtcNow.ToString("yyyyMMdd_HHmmss"); + var cacheFileName = $"{safeLocationKey}_{timestamp}.png"; + var cacheFilePath = Path.Combine(_cacheService.GetCacheDirectory(), cacheFileName); + + // Take screenshot of the map area only + await page.ScreenshotAsync(new PageScreenshotOptions + { + Path = cacheFilePath, + Clip = new Clip + { + X = boundingBox.X, + Y = boundingBox.Y, + Width = boundingBox.Width, + Height = boundingBox.Height + } + }); + + _logger.LogInformation("Screenshot saved to: {Path}", cacheFilePath); + + // Save metadata alongside the screenshot + await _cacheService.SaveMetadataAsync(cacheFilePath, lastUpdatedInfo, cancellationToken); + + return ResponseBuilder.CreateRadarScreenshotResponse(cacheFilePath, lastUpdatedInfo); + } + catch (Exception ex) + { + // Save error debug info if debug is enabled + await _debugService.SaveErrorDebugAsync(debugFolder, $"Exception: {ex.Message}\n\nStackTrace:\n{ex.StackTrace}", page, consoleMessages, networkRequests, cancellationToken); + _logger.LogError(ex, "Error during radar screenshot capture"); + throw; + } + } +} + diff --git a/Services/TimeParsingService.cs b/Services/TimeParsingService.cs new file mode 100644 index 0000000..9f60a9c --- /dev/null +++ b/Services/TimeParsingService.cs @@ -0,0 +1,254 @@ +using BomLocalService.Models; +using BomLocalService.Services.Interfaces; +using Microsoft.Playwright; +using System.Globalization; +using System.Text.RegularExpressions; + +namespace BomLocalService.Services; + +public class TimeParsingService : ITimeParsingService +{ + private readonly ILogger _logger; + private readonly TimeZoneInfo _timeZoneInfo; + + public TimeParsingService(ILogger logger, IConfiguration configuration) + { + _logger = logger; + var timezone = configuration.GetValue("Timezone") ?? "Australia/Brisbane"; + + try + { + _timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById(timezone); + _logger.LogInformation("Using timezone: {Timezone} ({DisplayName})", timezone, _timeZoneInfo.DisplayName); + } + catch (TimeZoneNotFoundException) + { + _logger.LogWarning("Timezone {Timezone} not found, defaulting to Australia/Brisbane", timezone); + _timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById("Australia/Brisbane"); + } + } + + /// + /// Extracts last updated information from the weather map page + /// + public async Task ExtractLastUpdatedInfoAsync(IPage page) + { + try + { + // Look for the weather metadata section using the correct selector + // HTML structure:
with aria-label="Last updated" + var metadataSection = page.Locator("section[data-testid='weatherMetadata'], section[aria-label='Last updated']").First; + + // Extract text from each div separately to preserve structure + // Structure:
Observations: X minutes ago, 9:40 pm AEST
at Gympie...
Forecast: ...
+ var lastUpdatedText = await page.EvaluateAsync(@"() => { + const section = document.querySelector('section[data-testid=""weatherMetadata""]') || + document.querySelector('section[aria-label=""Last updated""]'); + if (!section) return null; + + // Get all divs and join with newlines to preserve structure + const divs = section.querySelectorAll('div'); + return Array.from(divs).map(div => div.textContent.trim()).filter(text => text).join(' '); + }"); + + if (string.IsNullOrEmpty(lastUpdatedText)) + { + // Fallback: get all text content + lastUpdatedText = await metadataSection.TextContentAsync(); + } + + return ParseLastUpdatedText(lastUpdatedText ?? string.Empty); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to extract last updated information"); + return new LastUpdatedInfo + { + ObservationTime = DateTime.UtcNow, + ForecastTime = DateTime.UtcNow + }; + } + } + + /// + /// Parses last updated text to extract observation time, forecast time, weather station, and distance + /// + public LastUpdatedInfo ParseLastUpdatedText(string text) + { + var info = new LastUpdatedInfo(); + _logger.LogInformation("Parsing last updated text: {Text}", text); + + // Try to parse "Observations: 11 minutes ago, 8:20 pm AEST at Gympie weather station, 30 km from Pomona, QLD" + // And "Forecast: 41 minutes ago, 7:50 pm AEST" + // Prefer parsing the actual time string over "minutes ago" for accuracy + + // Parse observation time - improved regex to better capture time and timezone + // Stop timezone capture before "at" (e.g., "AEST at Gympie" or "AESTat Gympie" should capture just "AEST") + // Pattern: "Observations: X minutes ago, 9:40 pm AESTat Gympie..." (note: sometimes no space between AEST and "at") + // Capture timezone as 3-4 uppercase letters (AEST, AEDT, etc.) optionally followed by "at" + var observationMatch = Regex.Match(text, @"Observations:\s*(?:\d+\s*minutes?\s*ago)?[,\s]+([\d:]+(?:\s*[ap]m)?)\s+([A-Z]{3,4})(?:at|\s+at|,|$)", RegexOptions.IgnoreCase); + _logger.LogDebug("Observation regex match result: Success={Success}, Groups.Count={Count}", observationMatch.Success, observationMatch.Groups.Count); + if (observationMatch.Success && observationMatch.Groups.Count >= 3 && observationMatch.Groups[1].Success) + { + var timeStr = observationMatch.Groups[1].Value.Trim(); + var timezoneStr = observationMatch.Groups[2].Success ? observationMatch.Groups[2].Value.Trim() : null; + + // Clean up timezone string - remove "at" if it got concatenated (e.g., "AESTat" -> "AEST") + if (!string.IsNullOrEmpty(timezoneStr) && timezoneStr.EndsWith("at", StringComparison.OrdinalIgnoreCase) && timezoneStr.Length > 2) + { + timezoneStr = timezoneStr.Substring(0, timezoneStr.Length - 2); + } + + _logger.LogDebug("Extracted observation time string: '{TimeStr}', timezone: '{TzStr}'", timeStr, timezoneStr ?? "null"); + + if (TryParseTimeString(timeStr, timezoneStr, out var observationTime)) + { + info.ObservationTime = observationTime; + _logger.LogInformation("Successfully parsed observation time: {Time} UTC (from '{TimeStr}' {TzStr})", + observationTime, timeStr, timezoneStr ?? "default timezone"); + } + else + { + // Fallback to "minutes ago" if time parsing fails + _logger.LogWarning("Failed to parse observation time from '{TimeStr}', falling back to 'minutes ago'", timeStr); + var minutesAgoMatch = Regex.Match(text, @"Observations:\s*(\d+)\s*minutes?\s*ago", RegexOptions.IgnoreCase); + if (minutesAgoMatch.Success && int.TryParse(minutesAgoMatch.Groups[1].Value, out var minutesAgo)) + { + info.ObservationTime = DateTime.UtcNow.AddMinutes(-minutesAgo); + _logger.LogInformation("Used 'minutes ago' fallback: {MinutesAgo} minutes ago = {Time} UTC", + minutesAgo, info.ObservationTime); + } + } + } + + // Parse forecast time + var forecastMatch = Regex.Match(text, @"Forecast:\s*(?:\d+\s*minutes?\s*ago)?[,\s]*([\d:]+(?:\s*[ap]m)?)\s+([A-Z]+)(?:\s+at|$)", RegexOptions.IgnoreCase); + if (forecastMatch.Success && forecastMatch.Groups[1].Success) + { + var timeStr = forecastMatch.Groups[1].Value.Trim(); + var timezoneStr = forecastMatch.Groups[2].Success ? forecastMatch.Groups[2].Value.Trim() : null; + + if (TryParseTimeString(timeStr, timezoneStr, out var forecastTime)) + { + info.ForecastTime = forecastTime; + } + else + { + // Fallback to "minutes ago" if time parsing fails + var minutesAgoMatch = Regex.Match(text, @"Forecast:\s*(\d+)\s*minutes?\s*ago", RegexOptions.IgnoreCase); + if (minutesAgoMatch.Success && int.TryParse(minutesAgoMatch.Groups[1].Value, out var minutesAgo)) + { + info.ForecastTime = DateTime.UtcNow.AddMinutes(-minutesAgo); + } + } + } + + // Extract weather station name + var stationMatch = Regex.Match(text, @"at\s+([^,]+)\s+weather\s+station", RegexOptions.IgnoreCase); + if (stationMatch.Success) + { + info.WeatherStation = stationMatch.Groups[1].Value.Trim(); + } + + // Extract distance + var distanceMatch = Regex.Match(text, @"(\d+)\s*km\s+from", RegexOptions.IgnoreCase); + if (distanceMatch.Success) + { + info.Distance = $"{distanceMatch.Groups[1].Value} km"; + } + + return info; + } + + /// + /// Tries to parse a time string with optional timezone to UTC + /// + private bool TryParseTimeString(string timeStr, string? timezoneStr, out DateTime utcTime) + { + utcTime = DateTime.UtcNow; + + try + { + // Parse time formats like "8:20 pm", "8:20pm", "20:20" + var timeFormats = new[] { "h:mm tt", "h:mmtt", "HH:mm", "H:mm tt", "H:mmtt", "H:mm" }; + DateTime localTime = default; + + // Try parsing with various formats + bool parsed = false; + foreach (var format in timeFormats) + { + if (DateTime.TryParseExact(timeStr, format, CultureInfo.InvariantCulture, DateTimeStyles.None, out var parsedTime)) + { + localTime = parsedTime; + parsed = true; + break; + } + } + + if (!parsed) + { + // Fallback to general parsing + if (!DateTime.TryParse(timeStr, CultureInfo.InvariantCulture, DateTimeStyles.None, out localTime)) + { + return false; + } + } + + // Determine the timezone - use configured timezone if not specified or if it's AEST/AEDT + TimeZoneInfo tz = _timeZoneInfo; + if (!string.IsNullOrEmpty(timezoneStr)) + { + // AEST/AEDT are typically the same as Australia/Brisbane or Australia/Sydney + // Both Brisbane and Sydney use AEST/AEDT + if (timezoneStr.Contains("AEST", StringComparison.OrdinalIgnoreCase) || + timezoneStr.Contains("AEDT", StringComparison.OrdinalIgnoreCase)) + { + tz = _timeZoneInfo; // Use configured timezone + } + else + { + // Try to find the timezone, fallback to configured + try + { + tz = TimeZoneInfo.FindSystemTimeZoneById(timezoneStr); + } + catch + { + tz = _timeZoneInfo; + } + } + } + + // Get current time in the target timezone to determine the date + var nowInTz = TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, tz); + var dateTimeInTz = nowInTz.Date.Add(localTime.TimeOfDay); + + // If the parsed time is in the future (e.g., it's 9pm but we parsed 8pm and it's actually yesterday), + // or if it's more than 12 hours in the past, assume it's from yesterday + if (dateTimeInTz > nowInTz) + { + // Time is in the future, so it must be from yesterday + dateTimeInTz = dateTimeInTz.AddDays(-1); + } + else if ((nowInTz - dateTimeInTz).TotalHours > 12) + { + // More than 12 hours ago, likely from yesterday + dateTimeInTz = dateTimeInTz.AddDays(-1); + } + + // Convert to UTC + // Note: Brisbane (Australia/Brisbane) is always UTC+10 (AEST), no daylight saving + utcTime = TimeZoneInfo.ConvertTimeToUtc(dateTimeInTz, tz); + + _logger.LogInformation("Parsed time string '{TimeStr}' with timezone '{TzStr}' as {DateTimeInTz} local ({UtcTime} UTC)", + timeStr, timezoneStr ?? "default", dateTimeInTz, utcTime); + + return true; + } + catch + { + return false; + } + } +} + diff --git a/Utilities/FilePathHelper.cs b/Utilities/FilePathHelper.cs new file mode 100644 index 0000000..ffa3726 --- /dev/null +++ b/Utilities/FilePathHelper.cs @@ -0,0 +1,40 @@ +namespace BomLocalService.Utilities; + +public static class FilePathHelper +{ + /// + /// Gets the cache directory path from configuration or returns default + /// + public static string GetCacheDirectory(IConfiguration configuration) + { + return configuration.GetValue("CacheDirectory") + ?? Path.Combine(AppContext.BaseDirectory, "cache"); + } + + /// + /// Gets the metadata file path for a given screenshot path + /// + public static string GetMetadataFilePath(string screenshotPath) + { + return Path.ChangeExtension(screenshotPath, ".json"); + } + + /// + /// Gets the cache file pattern for a location key (e.g., "Pomona_QLD_*.png") + /// + public static string GetCacheFilePattern(string locationKey) + { + var safeLocationKey = LocationHelper.SanitizeFileName(locationKey); + return $"{safeLocationKey}_*.png"; + } + + /// + /// Gets the cache file pattern for suburb and state + /// + public static string GetCacheFilePattern(string suburb, string state) + { + var locationKey = LocationHelper.GetLocationKey(suburb, state); + return GetCacheFilePattern(locationKey); + } +} + diff --git a/Utilities/LocationHelper.cs b/Utilities/LocationHelper.cs new file mode 100644 index 0000000..5526bc4 --- /dev/null +++ b/Utilities/LocationHelper.cs @@ -0,0 +1,63 @@ +namespace BomLocalService.Utilities; + +public static class LocationHelper +{ + /// + /// Creates a location key from suburb and state (e.g., "Pomona_QLD") + /// + public static string GetLocationKey(string suburb, string state) + { + return $"{suburb}_{state}"; + } + + /// + /// Sanitizes a filename by removing invalid characters and replacing spaces/commas + /// + public static string SanitizeFileName(string fileName) + { + var invalidChars = Path.GetInvalidFileNameChars(); + return string.Join("_", fileName.Split(invalidChars, StringSplitOptions.RemoveEmptyEntries)) + .Replace(" ", "_") + .Replace(",", ""); + } + + /// + /// Parses suburb and state from a cache filename (format: Suburb_State_YYYYMMDD_HHMMSS.png) + /// Returns null if parsing fails + /// + public static (string suburb, string state)? ParseLocationFromFilename(string fileName) + { + var fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileName); + var parts = fileNameWithoutExtension.Split('_'); + + if (parts.Length >= 2) + { + return (parts[0], parts[1]); + } + + return null; + } + + /// + /// Parses timestamp from a cache filename (format: Suburb_State_YYYYMMDD_HHMMSS.png) + /// Returns null if parsing fails + /// + public static DateTime? ParseTimestampFromFilename(string fileName) + { + var fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileName); + var parts = fileNameWithoutExtension.Split('_'); + + if (parts.Length >= 3) + { + // Last two parts should be date and time + var dateTimeStr = $"{parts[^2]}_{parts[^1]}"; + if (DateTime.TryParseExact(dateTimeStr, "yyyyMMdd_HHmmss", null, System.Globalization.DateTimeStyles.None, out var fileTime)) + { + return fileTime; + } + } + + return null; + } +} + diff --git a/Utilities/ResponseBuilder.cs b/Utilities/ResponseBuilder.cs new file mode 100644 index 0000000..c228c8c --- /dev/null +++ b/Utilities/ResponseBuilder.cs @@ -0,0 +1,40 @@ +using BomLocalService.Models; + +namespace BomLocalService.Utilities; + +public static class ResponseBuilder +{ + /// + /// Creates a RadarScreenshotResponse from a file path and metadata + /// + public static RadarScreenshotResponse CreateRadarScreenshotResponse( + string imagePath, + LastUpdatedInfo? metadata = null) + { + var lastWriteTime = File.Exists(imagePath) + ? File.GetLastWriteTime(imagePath) + : DateTime.UtcNow; + + if (metadata == null) + { + return new RadarScreenshotResponse + { + ImagePath = imagePath, + LastUpdated = lastWriteTime, + ObservationTime = DateTime.UtcNow, + ForecastTime = DateTime.UtcNow + }; + } + + return new RadarScreenshotResponse + { + ImagePath = imagePath, + LastUpdated = lastWriteTime, + ObservationTime = metadata.ObservationTime, + ForecastTime = metadata.ForecastTime, + WeatherStation = metadata.WeatherStation, + Distance = metadata.Distance + }; + } +} + diff --git a/Utilities/SelectorHelper.cs b/Utilities/SelectorHelper.cs new file mode 100644 index 0000000..b38f9ef --- /dev/null +++ b/Utilities/SelectorHelper.cs @@ -0,0 +1,37 @@ +using Microsoft.Playwright; + +namespace BomLocalService.Utilities; + +public static class SelectorHelper +{ + /// + /// Tries to find a locator by trying multiple selectors in order + /// Returns the first visible locator found, or null if none found + /// + public static async Task FindLocatorBySelectorsAsync( + IPage page, + string[] selectors, + ILogger? logger = null) + { + foreach (var selector in selectors) + { + try + { + var locator = page.Locator(selector).First; + if (await locator.IsVisibleAsync()) + { + logger?.LogInformation("Found element with selector: {Selector}", selector); + return locator; + } + } + catch + { + // Continue to next selector + continue; + } + } + + return null; + } +} + diff --git a/Utilities/StateAbbreviationHelper.cs b/Utilities/StateAbbreviationHelper.cs new file mode 100644 index 0000000..c4c6e4b --- /dev/null +++ b/Utilities/StateAbbreviationHelper.cs @@ -0,0 +1,45 @@ +namespace BomLocalService.Utilities; + +public static class StateAbbreviationHelper +{ + private static readonly Dictionary StateMappings = new(StringComparer.OrdinalIgnoreCase) + { + { "qld", new[] { "queensland" } }, + { "nsw", new[] { "new south wales" } }, + { "vic", new[] { "victoria" } }, + { "sa", new[] { "south australia" } }, + { "wa", new[] { "western australia" } }, + { "tas", new[] { "tasmania" } }, + { "nt", new[] { "northern territory" } }, + { "act", new[] { "australian capital territory" } } + }; + + /// + /// Checks if the text matches the state abbreviation or full state name + /// + public static bool MatchesState(string text, string stateAbbreviation) + { + if (string.IsNullOrWhiteSpace(stateAbbreviation)) + { + return true; // No state specified, match any + } + + var textLower = text.ToLower(); + var stateLower = stateAbbreviation.ToLower().Trim(); + + // Direct match + if (textLower.Contains(stateLower)) + { + return true; + } + + // Check abbreviation mappings + if (StateMappings.TryGetValue(stateLower, out var fullNames)) + { + return fullNames.Any(fullName => textLower.Contains(fullName)); + } + + return false; + } +} + diff --git a/Utilities/ValidationHelper.cs b/Utilities/ValidationHelper.cs new file mode 100644 index 0000000..769587f --- /dev/null +++ b/Utilities/ValidationHelper.cs @@ -0,0 +1,24 @@ +namespace BomLocalService.Utilities; + +public static class ValidationHelper +{ + /// + /// Validates suburb and state parameters + /// Returns error message if validation fails, null if valid + /// + public static string? ValidateLocation(string suburb, string state) + { + if (string.IsNullOrWhiteSpace(suburb)) + { + return "Suburb is required"; + } + + if (string.IsNullOrWhiteSpace(state)) + { + return "State is required"; + } + + return null; + } +} + diff --git a/appsettings.json b/appsettings.json new file mode 100644 index 0000000..4260920 --- /dev/null +++ b/appsettings.json @@ -0,0 +1,30 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*", + "CacheDirectory": "./cache", + "CacheRetentionHours": 24, + "CacheExpirationMinutes": 12.5, + "CacheManagement": { + "CheckIntervalMinutes": 5, + "InitialDelaySeconds": 10, + "UpdateStaggerSeconds": 2, + "LocationStaggerSeconds": 1 + }, + "CacheCleanup": { + "IntervalHours": 1 + }, + "Screenshot": { + "DynamicContentWaitMs": 2000, + "TileRenderWaitMs": 5000 + }, + "Debug": { + "Enabled": false, + "WaitMs": 2000 + }, + "Timezone": "Australia/Brisbane" +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..3446d29 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,45 @@ +services: + bom-local-service: + build: + context: . + dockerfile: Dockerfile + container_name: bom-local-service + ports: + - "${HOST_PORT:-8082}:${CONTAINER_PORT:-8080}" + volumes: + - ./cache:/app/cache + environment: + # ASP.NET Core configuration + # Default to Production for security (use Development only for local debugging) + - ASPNETCORE_ENVIRONMENT=${ASPNETCORE_ENVIRONMENT:-Production} + - ASPNETCORE_URLS=http://+:${CONTAINER_PORT:-8080} + + # Virtual display for non-headless browser + - DISPLAY=:99 + + # Application configuration (override appsettings.json) + # Use double underscore (__) for nested config keys + - CACHEDIRECTORY=${CACHEDIRECTORY:-/app/cache} + - CACHERETENTIONHOURS=${CACHERETENTIONHOURS:-24} + - CACHEEXPIRATIONMINUTES=${CACHEEXPIRATIONMINUTES:-12.5} + - CACHEMANAGEMENT__CHECKINTERVALMINUTES=${CACHEMANAGEMENT__CHECKINTERVALMINUTES:-5} + - CACHEMANAGEMENT__INITIALDELAYSECONDS=${CACHEMANAGEMENT__INITIALDELAYSECONDS:-10} + - CACHEMANAGEMENT__UPDATESTAGGERSECONDS=${CACHEMANAGEMENT__UPDATESTAGGERSECONDS:-2} + - CACHEMANAGEMENT__LOCATIONSTAGGERSECONDS=${CACHEMANAGEMENT__LOCATIONSTAGGERSECONDS:-1} + - CACHECLEANUP__INTERVALHOURS=${CACHECLEANUP__INTERVALHOURS:-1} + - SCREENSHOT__DYNAMICCONTENTWAITMS=${SCREENSHOT__DYNAMICCONTENTWAITMS:-2000} + - SCREENSHOT__TILERENDERWAITMS=${SCREENSHOT__TILERENDERWAITMS:-5000} + - DEBUG__ENABLED=${DEBUG__ENABLED:-false} + - DEBUG__WAITMS=${DEBUG__WAITMS:-2000} + - TIMEZONE=${TIMEZONE:-Australia/Brisbane} + shm_size: '1gb' + ipc: host + restart: unless-stopped + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] +