mirror of
https://github.com/alexhopeoconnor/bom-local-service.git
synced 2026-10-04 03:28:11 +10:00
Initial commit: BOM Local Service
This commit is contained in:
@@ -0,0 +1,27 @@
|
|||||||
|
**/.classpath
|
||||||
|
**/.dockerignore
|
||||||
|
**/.env
|
||||||
|
**/.git
|
||||||
|
**/.gitignore
|
||||||
|
**/.project
|
||||||
|
**/.settings
|
||||||
|
**/.toolstarget
|
||||||
|
**/.vs
|
||||||
|
**/.vscode
|
||||||
|
**/.idea
|
||||||
|
**/*.*proj.user
|
||||||
|
**/*.dbmdl
|
||||||
|
**/*.jfm
|
||||||
|
**/azds.yaml
|
||||||
|
**/bin
|
||||||
|
**/charts
|
||||||
|
**/docker-compose*
|
||||||
|
**/Dockerfile*
|
||||||
|
**/node_modules
|
||||||
|
**/npm-debug.log
|
||||||
|
**/obj
|
||||||
|
**/secrets.dev.yaml
|
||||||
|
**/values.dev.yaml
|
||||||
|
LICENSE
|
||||||
|
README.md
|
||||||
|
|
||||||
+78
@@ -0,0 +1,78 @@
|
|||||||
|
## .NET build artifacts
|
||||||
|
bin/
|
||||||
|
obj/
|
||||||
|
*.dll
|
||||||
|
*.exe
|
||||||
|
*.pdb
|
||||||
|
*.cache
|
||||||
|
*.deps.json
|
||||||
|
*.runtimeconfig.json
|
||||||
|
*.staticwebassets.endpoints.json
|
||||||
|
|
||||||
|
## Build results
|
||||||
|
[Dd]ebug/
|
||||||
|
[Rr]elease/
|
||||||
|
[Bb]uild/
|
||||||
|
x64/
|
||||||
|
x86/
|
||||||
|
[Aa]rm/
|
||||||
|
[Aa]rm64/
|
||||||
|
bld/
|
||||||
|
[Bb]in/
|
||||||
|
[Oo]bj/
|
||||||
|
[Ll]og/
|
||||||
|
[Ll]ogs/
|
||||||
|
|
||||||
|
## Visual Studio / Rider
|
||||||
|
.vs/
|
||||||
|
.idea/
|
||||||
|
*.user
|
||||||
|
*.suo
|
||||||
|
*.userosscache
|
||||||
|
*.sln.docstates
|
||||||
|
*.userprefs
|
||||||
|
|
||||||
|
## VS Code
|
||||||
|
.vscode/
|
||||||
|
*.code-workspace
|
||||||
|
|
||||||
|
## Cache directory (generated screenshots and metadata)
|
||||||
|
cache/
|
||||||
|
|
||||||
|
## NuGet
|
||||||
|
*.nupkg
|
||||||
|
*.snupkg
|
||||||
|
**/packages/*
|
||||||
|
!**/packages/build/
|
||||||
|
project.lock.json
|
||||||
|
project.fragment.lock.json
|
||||||
|
artifacts/
|
||||||
|
|
||||||
|
## Test results
|
||||||
|
[Tt]est[Rr]esult*/
|
||||||
|
[Bb]uild[Ll]og.*
|
||||||
|
*.trx
|
||||||
|
*.coverage
|
||||||
|
*.coveragexml
|
||||||
|
|
||||||
|
## OS files
|
||||||
|
.DS_Store
|
||||||
|
.DS_Store?
|
||||||
|
._*
|
||||||
|
.Spotlight-V100
|
||||||
|
.Trashes
|
||||||
|
ehthumbs.db
|
||||||
|
Thumbs.db
|
||||||
|
Desktop.ini
|
||||||
|
|
||||||
|
## Temporary files
|
||||||
|
*.tmp
|
||||||
|
*.temp
|
||||||
|
*.swp
|
||||||
|
*~
|
||||||
|
.*.sw?
|
||||||
|
|
||||||
|
## Docker
|
||||||
|
# Keep Dockerfile and docker-compose.yml, ignore temporary Docker files
|
||||||
|
*.dockerignore.bak
|
||||||
|
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<TargetFramework>net9.0</TargetFramework>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="9.0.11" />
|
||||||
|
<PackageReference Include="Microsoft.Playwright" Version="1.57.0" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
</Project>
|
||||||
@@ -0,0 +1,163 @@
|
|||||||
|
using BomLocalService.Models;
|
||||||
|
using BomLocalService.Services.Interfaces;
|
||||||
|
using BomLocalService.Utilities;
|
||||||
|
using Microsoft.AspNetCore.Mvc;
|
||||||
|
|
||||||
|
namespace BomLocalService.Controllers;
|
||||||
|
|
||||||
|
[ApiController]
|
||||||
|
[Route("api/radar")]
|
||||||
|
public class RadarController : ControllerBase
|
||||||
|
{
|
||||||
|
private readonly IBomRadarService _bomRadarService;
|
||||||
|
private readonly ILogger<RadarController> _logger;
|
||||||
|
|
||||||
|
public RadarController(IBomRadarService bomRadarService, ILogger<RadarController> logger)
|
||||||
|
{
|
||||||
|
_bomRadarService = bomRadarService;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Get radar screenshot for a location. Returns cached screenshot if available, or triggers background update.
|
||||||
|
/// </summary>
|
||||||
|
[HttpGet("{suburb}/{state}")]
|
||||||
|
public async Task<ActionResult> GetScreenshot(string suburb, string state, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var validationError = ValidationHelper.ValidateLocation(suburb, state);
|
||||||
|
if (validationError != null)
|
||||||
|
{
|
||||||
|
return BadRequest(new { error = validationError });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if cache needs updating and trigger if needed (non-blocking)
|
||||||
|
_ = Task.Run(async () =>
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await _bomRadarService.TriggerCacheUpdateAsync(suburb, state, CancellationToken.None);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(ex, "Background cache update failed for {Suburb}, {State}", suburb, state);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Return cached screenshot if it exists (no waiting)
|
||||||
|
var result = await _bomRadarService.GetCachedScreenshotAsync(suburb, state, cancellationToken);
|
||||||
|
|
||||||
|
if (result == null)
|
||||||
|
{
|
||||||
|
return NotFound(new {
|
||||||
|
error = "Screenshot not found in cache. Cache update has been triggered in background. Please retry in a few moments.",
|
||||||
|
retryAfter = 30, // seconds
|
||||||
|
refreshEndpoint = $"/api/radar/{suburb}/{state}/refresh"
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return the image file
|
||||||
|
if (System.IO.File.Exists(result.ImagePath))
|
||||||
|
{
|
||||||
|
var imageBytes = await System.IO.File.ReadAllBytesAsync(result.ImagePath, cancellationToken);
|
||||||
|
var contentType = "image/png";
|
||||||
|
return File(imageBytes, contentType, Path.GetFileName(result.ImagePath));
|
||||||
|
}
|
||||||
|
|
||||||
|
return NotFound(new { error = "Screenshot file not found" });
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Error getting cached radar screenshot for suburb: {Suburb}, state: {State}", suburb, state);
|
||||||
|
return StatusCode(500, new { error = "An error occurred while getting the radar screenshot", message = ex.Message });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Get metadata about the cached radar data for a location (observation time, forecast time, etc.)
|
||||||
|
/// </summary>
|
||||||
|
[HttpGet("{suburb}/{state}/metadata")]
|
||||||
|
public async Task<ActionResult<LastUpdatedInfo>> GetMetadata(string suburb, string state, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var validationError = ValidationHelper.ValidateLocation(suburb, state);
|
||||||
|
if (validationError != null)
|
||||||
|
{
|
||||||
|
return BadRequest(new { error = validationError });
|
||||||
|
}
|
||||||
|
|
||||||
|
var result = await _bomRadarService.GetLastUpdatedInfoAsync(suburb, state, cancellationToken);
|
||||||
|
|
||||||
|
if (result == null)
|
||||||
|
{
|
||||||
|
return NotFound(new { error = "No cached data found. Use POST /api/radar/{suburb}/{state}/refresh to trigger cache update." });
|
||||||
|
}
|
||||||
|
|
||||||
|
return Ok(result);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Error getting metadata for suburb: {Suburb}, state: {State}", suburb, state);
|
||||||
|
return StatusCode(500, new { error = "An error occurred while getting metadata", message = ex.Message });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Manually trigger a cache refresh for a location. Returns status of the update operation.
|
||||||
|
/// </summary>
|
||||||
|
[HttpPost("{suburb}/{state}/refresh")]
|
||||||
|
public async Task<ActionResult<CacheUpdateStatus>> RefreshCache(string suburb, string state, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var validationError = ValidationHelper.ValidateLocation(suburb, state);
|
||||||
|
if (validationError != null)
|
||||||
|
{
|
||||||
|
return BadRequest(new { error = validationError });
|
||||||
|
}
|
||||||
|
|
||||||
|
var status = await _bomRadarService.TriggerCacheUpdateAsync(suburb, state, cancellationToken);
|
||||||
|
return Ok(status);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Error triggering cache update for suburb: {Suburb}, state: {State}", suburb, state);
|
||||||
|
return StatusCode(500, new { error = "An error occurred while triggering cache update", message = ex.Message });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Delete cached data for a location.
|
||||||
|
/// </summary>
|
||||||
|
[HttpDelete("{suburb}/{state}")]
|
||||||
|
public async Task<ActionResult> DeleteCache(string suburb, string state, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var validationError = ValidationHelper.ValidateLocation(suburb, state);
|
||||||
|
if (validationError != null)
|
||||||
|
{
|
||||||
|
return BadRequest(new { error = validationError });
|
||||||
|
}
|
||||||
|
|
||||||
|
var deleted = await _bomRadarService.DeleteCachedLocationAsync(suburb, state, cancellationToken);
|
||||||
|
|
||||||
|
if (deleted)
|
||||||
|
{
|
||||||
|
return Ok(new { message = $"Cache deleted for {suburb}, {state}" });
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
return NotFound(new { error = $"No cached data found for {suburb}, {state}" });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Error deleting cached location for suburb: {Suburb}, state: {State}", suburb, state);
|
||||||
|
return StatusCode(500, new { error = "An error occurred while deleting cached location", message = ex.Message });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
+46
@@ -0,0 +1,46 @@
|
|||||||
|
# Build stage
|
||||||
|
FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build
|
||||||
|
WORKDIR /src
|
||||||
|
|
||||||
|
# Copy csproj and restore dependencies
|
||||||
|
COPY ["BomLocalService.csproj", "./"]
|
||||||
|
RUN dotnet restore "BomLocalService.csproj"
|
||||||
|
|
||||||
|
# Copy everything else and build
|
||||||
|
COPY . .
|
||||||
|
RUN dotnet build "BomLocalService.csproj" -c Release -o /app/build
|
||||||
|
|
||||||
|
# Publish stage
|
||||||
|
FROM build AS publish
|
||||||
|
RUN dotnet publish "BomLocalService.csproj" -c Release -o /app/publish /p:UseAppHost=false
|
||||||
|
|
||||||
|
# Runtime stage - use official Playwright .NET image that has browsers and dependencies pre-installed
|
||||||
|
FROM mcr.microsoft.com/playwright/dotnet:v1.57.0-noble AS final
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Install .NET 9 runtime and Xvfb for virtual display (Playwright image has browsers but may need .NET 9)
|
||||||
|
RUN apt-get update && \
|
||||||
|
wget https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh && \
|
||||||
|
chmod +x dotnet-install.sh && \
|
||||||
|
./dotnet-install.sh --channel 9.0 --install-dir /usr/share/dotnet && \
|
||||||
|
rm dotnet-install.sh && \
|
||||||
|
apt-get install -y xvfb x11vnc fluxbox && \
|
||||||
|
apt-get clean && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Copy published app
|
||||||
|
COPY --from=publish /app/publish .
|
||||||
|
|
||||||
|
# Create cache directory for screenshots
|
||||||
|
RUN mkdir -p /app/cache && chmod 777 /app/cache
|
||||||
|
|
||||||
|
# Create startup script to run Xvfb and then the app
|
||||||
|
RUN echo '#!/bin/bash\n\
|
||||||
|
Xvfb :99 -screen 0 1920x1080x24 > /dev/null 2>&1 &\n\
|
||||||
|
export DISPLAY=:99\n\
|
||||||
|
exec dotnet BomLocalService.dll' > /app/start.sh && \
|
||||||
|
chmod +x /app/start.sh
|
||||||
|
|
||||||
|
EXPOSE 8080
|
||||||
|
|
||||||
|
ENTRYPOINT ["/app/start.sh"]
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
namespace BomLocalService.Models;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Status information about a cache update operation for a location.
|
||||||
|
/// Returned when manually triggering a cache refresh via the refresh endpoint.
|
||||||
|
/// </summary>
|
||||||
|
public class CacheUpdateStatus
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Indicates whether a background cache update was triggered.
|
||||||
|
/// True if the cache was missing or stale and an update was initiated.
|
||||||
|
/// False if the cache is valid and no update was needed.
|
||||||
|
/// </summary>
|
||||||
|
public bool UpdateTriggered { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Indicates whether cached screenshot files exist for this location.
|
||||||
|
/// True if PNG screenshot files are found in the cache directory.
|
||||||
|
/// False if no cached files exist for this location.
|
||||||
|
/// </summary>
|
||||||
|
public bool CacheExists { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Indicates whether the existing cache is still valid (not expired).
|
||||||
|
/// Cache is considered valid if the observation time plus expiration buffer (typically 15.5 minutes)
|
||||||
|
/// is still in the future. BOM updates observations every 15 minutes, so cache expiration
|
||||||
|
/// includes a small buffer to account for timing variations.
|
||||||
|
/// True if cache exists and is still valid, false if cache is missing or expired.
|
||||||
|
/// </summary>
|
||||||
|
public bool CacheIsValid { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The UTC timestamp when the current cache will expire and should be refreshed.
|
||||||
|
/// Calculated as: ObservationTime + CacheExpirationMinutes (typically observation time + 15.5 minutes).
|
||||||
|
/// Null if no cache exists or metadata is not available.
|
||||||
|
/// </summary>
|
||||||
|
public DateTime? CacheExpiresAt { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The UTC timestamp when the next cache update should occur.
|
||||||
|
/// If cache is valid: equals CacheExpiresAt (when current cache expires).
|
||||||
|
/// If cache is invalid/missing: equals current time + CacheExpirationMinutes (when background update will complete).
|
||||||
|
/// Null if cache is valid and CacheExpiresAt is null.
|
||||||
|
/// </summary>
|
||||||
|
public DateTime? NextUpdateTime { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Human-readable message describing the cache status and action taken.
|
||||||
|
/// Possible values:
|
||||||
|
/// - "Cache is valid, no update needed" - Cache exists and is still fresh
|
||||||
|
/// - "Cache is stale, update triggered" - Cache exists but expired, update initiated
|
||||||
|
/// - "No cache exists, update triggered" - No cache found, update initiated
|
||||||
|
/// </summary>
|
||||||
|
public string? Message { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
namespace BomLocalService.Models;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Metadata extracted from the BOM weather map page about observation and forecast times.
|
||||||
|
/// This information is scraped from the weather metadata section on the BOM website.
|
||||||
|
/// </summary>
|
||||||
|
public class LastUpdatedInfo
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// The UTC timestamp when the weather observations were taken.
|
||||||
|
/// Extracted from the BOM website metadata section (e.g., "Observations: 11 minutes ago, 8:20 pm AEST").
|
||||||
|
/// BOM updates observations every 15 minutes (at :00, :15, :30, :45 past the hour).
|
||||||
|
/// </summary>
|
||||||
|
public DateTime ObservationTime { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The UTC timestamp when the weather forecast was generated.
|
||||||
|
/// Extracted from the BOM website metadata section (e.g., "Forecast: 41 minutes ago, 7:50 pm AEST").
|
||||||
|
/// Forecasts are typically updated less frequently than observations.
|
||||||
|
/// </summary>
|
||||||
|
public DateTime ForecastTime { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The name of the weather station providing the observations for this location.
|
||||||
|
/// Extracted from text like "at Gympie weather station" in the metadata section.
|
||||||
|
/// May be null if the station name cannot be parsed from the page.
|
||||||
|
/// Example: "Gympie", "Brisbane", "Sydney"
|
||||||
|
/// </summary>
|
||||||
|
public string? WeatherStation { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The distance from the requested location to the weather station.
|
||||||
|
/// Extracted from text like "30 km from Pomona, QLD" in the metadata section.
|
||||||
|
/// Format: "{number} km" (e.g., "30 km", "15 km").
|
||||||
|
/// May be null if the distance cannot be parsed from the page.
|
||||||
|
/// </summary>
|
||||||
|
public string? Distance { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
namespace BomLocalService.Models;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Response model containing the radar screenshot image path and associated metadata.
|
||||||
|
/// This is the primary response returned when requesting a radar screenshot for a location.
|
||||||
|
/// </summary>
|
||||||
|
public class RadarScreenshotResponse
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// The full file system path to the cached PNG screenshot image file.
|
||||||
|
/// This is the path on the server where the screenshot is stored.
|
||||||
|
/// Format: "{CacheDirectory}/{Suburb}_{State}_{Timestamp}.png"
|
||||||
|
/// Example: "/app/cache/Pomona_QLD_20251207_000906.png"
|
||||||
|
/// </summary>
|
||||||
|
public string ImagePath { get; set; } = string.Empty;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The UTC timestamp when the screenshot file was last written/modified on disk.
|
||||||
|
/// This is the file system modification time, which typically matches when the screenshot was captured.
|
||||||
|
/// Used as a fallback if metadata is not available.
|
||||||
|
/// </summary>
|
||||||
|
public DateTime LastUpdated { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The UTC timestamp when the weather observations were taken (from BOM website).
|
||||||
|
/// This comes from the LastUpdatedInfo metadata extracted from the BOM weather map page.
|
||||||
|
/// BOM updates observations every 15 minutes (at :00, :15, :30, :45 past the hour).
|
||||||
|
/// If metadata is not available, defaults to DateTime.UtcNow.
|
||||||
|
/// </summary>
|
||||||
|
public DateTime ObservationTime { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The UTC timestamp when the weather forecast was generated (from BOM website).
|
||||||
|
/// This comes from the LastUpdatedInfo metadata extracted from the BOM weather map page.
|
||||||
|
/// Forecasts are typically updated less frequently than observations.
|
||||||
|
/// If metadata is not available, defaults to DateTime.UtcNow.
|
||||||
|
/// </summary>
|
||||||
|
public DateTime ForecastTime { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The name of the weather station providing the observations for this location.
|
||||||
|
/// Extracted from the BOM website metadata (e.g., "Gympie", "Brisbane").
|
||||||
|
/// May be null if the station name cannot be parsed or if metadata is not available.
|
||||||
|
/// </summary>
|
||||||
|
public string? WeatherStation { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The distance from the requested location to the weather station.
|
||||||
|
/// Extracted from the BOM website metadata (e.g., "30 km", "15 km").
|
||||||
|
/// Format: "{number} km".
|
||||||
|
/// May be null if the distance cannot be parsed or if metadata is not available.
|
||||||
|
/// </summary>
|
||||||
|
public string? Distance { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
+51
@@ -0,0 +1,51 @@
|
|||||||
|
using BomLocalService.Services;
|
||||||
|
using BomLocalService.Services.Interfaces;
|
||||||
|
|
||||||
|
var builder = WebApplication.CreateBuilder(args);
|
||||||
|
|
||||||
|
// Add services to the container
|
||||||
|
builder.Services.AddControllers();
|
||||||
|
builder.Services.AddEndpointsApiExplorer();
|
||||||
|
builder.Services.AddOpenApi();
|
||||||
|
|
||||||
|
// Register core services via interfaces (order matters - dependencies must be registered first)
|
||||||
|
builder.Services.AddSingleton<IDebugService, DebugService>();
|
||||||
|
builder.Services.AddSingleton<ICacheService, CacheService>();
|
||||||
|
builder.Services.AddSingleton<ITimeParsingService, TimeParsingService>();
|
||||||
|
builder.Services.AddSingleton<IBrowserService, BrowserService>();
|
||||||
|
builder.Services.AddSingleton<IScrapingService, ScrapingService>();
|
||||||
|
|
||||||
|
// Register BOM Radar Service as singleton (orchestrator, depends on all above services)
|
||||||
|
builder.Services.AddSingleton<IBomRadarService, BomLocalService.Services.BomRadarService>();
|
||||||
|
|
||||||
|
// Register background services (order matters - management service needs radar service)
|
||||||
|
builder.Services.AddHostedService<CacheCleanupService>();
|
||||||
|
builder.Services.AddHostedService<CacheManagementService>();
|
||||||
|
|
||||||
|
// Add configuration - appsettings.json provides default values
|
||||||
|
// All values can be overridden via environment variables
|
||||||
|
builder.Configuration.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true);
|
||||||
|
|
||||||
|
var app = builder.Build();
|
||||||
|
|
||||||
|
// Configure the HTTP request pipeline
|
||||||
|
if (app.Environment.IsDevelopment())
|
||||||
|
{
|
||||||
|
app.MapOpenApi();
|
||||||
|
}
|
||||||
|
|
||||||
|
app.UseHttpsRedirection();
|
||||||
|
app.UseAuthorization();
|
||||||
|
app.MapControllers();
|
||||||
|
|
||||||
|
// Cleanup on shutdown
|
||||||
|
app.Lifetime.ApplicationStopped.Register(() =>
|
||||||
|
{
|
||||||
|
var service = app.Services.GetService<IBomRadarService>();
|
||||||
|
if (service is IDisposable disposable)
|
||||||
|
{
|
||||||
|
disposable.Dispose();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
app.Run();
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://json.schemastore.org/launchsettings.json",
|
||||||
|
"profiles": {
|
||||||
|
"http": {
|
||||||
|
"commandName": "Project",
|
||||||
|
"dotnetRunMessages": true,
|
||||||
|
"launchBrowser": false,
|
||||||
|
"applicationUrl": "http://localhost:5249",
|
||||||
|
"environmentVariables": {
|
||||||
|
"ASPNETCORE_ENVIRONMENT": "Development"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"https": {
|
||||||
|
"commandName": "Project",
|
||||||
|
"dotnetRunMessages": true,
|
||||||
|
"launchBrowser": false,
|
||||||
|
"applicationUrl": "https://localhost:7190;http://localhost:5249",
|
||||||
|
"environmentVariables": {
|
||||||
|
"ASPNETCORE_ENVIRONMENT": "Development"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,217 @@
|
|||||||
|
using BomLocalService.Models;
|
||||||
|
using BomLocalService.Services.Interfaces;
|
||||||
|
using BomLocalService.Utilities;
|
||||||
|
using Microsoft.Playwright;
|
||||||
|
|
||||||
|
namespace BomLocalService.Services;
|
||||||
|
|
||||||
|
public class BomRadarService : IBomRadarService, IDisposable
|
||||||
|
{
|
||||||
|
private readonly ILogger<BomRadarService> _logger;
|
||||||
|
private readonly ICacheService _cacheService;
|
||||||
|
private readonly IBrowserService _browserService;
|
||||||
|
private readonly IScrapingService _scrapingService;
|
||||||
|
private readonly IDebugService _debugService;
|
||||||
|
private readonly double _cacheExpirationMinutes;
|
||||||
|
|
||||||
|
public BomRadarService(
|
||||||
|
ILogger<BomRadarService> logger,
|
||||||
|
ICacheService cacheService,
|
||||||
|
IBrowserService browserService,
|
||||||
|
IScrapingService scrapingService,
|
||||||
|
IDebugService debugService,
|
||||||
|
IConfiguration configuration)
|
||||||
|
{
|
||||||
|
_logger = logger;
|
||||||
|
_cacheService = cacheService;
|
||||||
|
_browserService = browserService;
|
||||||
|
_scrapingService = scrapingService;
|
||||||
|
_debugService = debugService;
|
||||||
|
_cacheExpirationMinutes = configuration.GetValue<double>("CacheExpirationMinutes", 15.5);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<RadarScreenshotResponse?> GetCachedScreenshotAsync(string suburb, string state, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
var (cachedPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
|
||||||
|
|
||||||
|
if (string.IsNullOrEmpty(cachedPath) || !File.Exists(cachedPath))
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return ResponseBuilder.CreateRadarScreenshotResponse(cachedPath, cachedMetadata);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<CacheUpdateStatus> TriggerCacheUpdateAsync(string suburb, string state, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
var status = new CacheUpdateStatus();
|
||||||
|
var (cachedPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
|
||||||
|
|
||||||
|
status.CacheExists = !string.IsNullOrEmpty(cachedPath) && File.Exists(cachedPath);
|
||||||
|
|
||||||
|
if (cachedMetadata != null)
|
||||||
|
{
|
||||||
|
status.CacheIsValid = _cacheService.IsCacheValid(cachedMetadata);
|
||||||
|
status.CacheExpiresAt = cachedMetadata.ObservationTime.AddMinutes(_cacheExpirationMinutes);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if we need to update
|
||||||
|
bool needsUpdate = !status.CacheExists || !status.CacheIsValid;
|
||||||
|
|
||||||
|
if (needsUpdate)
|
||||||
|
{
|
||||||
|
// Trigger async update (fire and forget)
|
||||||
|
_ = Task.Run(async () =>
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await FetchAndCacheScreenshotAsync(suburb, state, CancellationToken.None);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Error during background cache update for {Suburb}, {State}", suburb, state);
|
||||||
|
}
|
||||||
|
}, cancellationToken);
|
||||||
|
|
||||||
|
status.UpdateTriggered = true;
|
||||||
|
status.Message = status.CacheExists
|
||||||
|
? "Cache is stale, update triggered"
|
||||||
|
: "No cache exists, update triggered";
|
||||||
|
|
||||||
|
// Calculate next update time
|
||||||
|
if (status.CacheExpiresAt.HasValue && status.CacheExpiresAt.Value > DateTime.UtcNow)
|
||||||
|
{
|
||||||
|
status.NextUpdateTime = status.CacheExpiresAt.Value;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
status.NextUpdateTime = DateTime.UtcNow.AddMinutes(_cacheExpirationMinutes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
status.UpdateTriggered = false;
|
||||||
|
status.Message = "Cache is valid, no update needed";
|
||||||
|
status.NextUpdateTime = status.CacheExpiresAt;
|
||||||
|
}
|
||||||
|
|
||||||
|
return status;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<RadarScreenshotResponse> FetchAndCacheScreenshotAsync(string suburb, string state, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
_logger.LogInformation("Getting radar screenshot for suburb: {Suburb}, state: {State}", suburb, state);
|
||||||
|
|
||||||
|
// Check cache FIRST, before acquiring semaphore (cached requests shouldn't block)
|
||||||
|
var (cachedPath, cachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
|
||||||
|
|
||||||
|
if (!string.IsNullOrEmpty(cachedPath) && File.Exists(cachedPath) && cachedMetadata != null)
|
||||||
|
{
|
||||||
|
var isValid = _cacheService.IsCacheValid(cachedMetadata);
|
||||||
|
if (isValid)
|
||||||
|
{
|
||||||
|
_logger.LogInformation("Returning valid cached screenshot for {Suburb}, {State} (no semaphore needed)", suburb, state);
|
||||||
|
return ResponseBuilder.CreateRadarScreenshotResponse(cachedPath, cachedMetadata);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
var nextUpdate = cachedMetadata.ObservationTime.AddMinutes(_cacheExpirationMinutes);
|
||||||
|
var timeUntilExpiry = nextUpdate - DateTime.UtcNow;
|
||||||
|
_logger.LogInformation("Cached screenshot exists but is stale (observation time: {ObservationTime}, expired {TimeAgo} ago), fetching new one",
|
||||||
|
cachedMetadata.ObservationTime, -timeUntilExpiry);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (!string.IsNullOrEmpty(cachedPath) && File.Exists(cachedPath))
|
||||||
|
{
|
||||||
|
_logger.LogWarning("Cached screenshot found but no metadata file exists, fetching new one");
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_logger.LogInformation("No cached screenshot found for {Suburb}, {State}, fetching new one", suburb, state);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only acquire semaphore if we need to fetch new screenshot
|
||||||
|
var semaphore = _browserService.GetSemaphore();
|
||||||
|
await semaphore.WaitAsync(cancellationToken);
|
||||||
|
|
||||||
|
// Generate request ID for debugging
|
||||||
|
var requestId = $"{DateTime.UtcNow:yyyyMMdd_HHmmss}_{Guid.NewGuid():N}";
|
||||||
|
var debugFolder = _debugService.CreateRequestFolder(requestId);
|
||||||
|
|
||||||
|
IBrowserContext? context = null;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
// Double-check cache after acquiring semaphore (another request might have just created it)
|
||||||
|
var (recheckCachedPath, recheckCachedMetadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
|
||||||
|
if (!string.IsNullOrEmpty(recheckCachedPath) && File.Exists(recheckCachedPath) && recheckCachedMetadata != null && _cacheService.IsCacheValid(recheckCachedMetadata))
|
||||||
|
{
|
||||||
|
_logger.LogInformation("Cache became valid while waiting for semaphore, returning cached screenshot");
|
||||||
|
return ResponseBuilder.CreateRadarScreenshotResponse(recheckCachedPath, recheckCachedMetadata);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Need to capture new screenshot
|
||||||
|
context = await _browserService.CreateContextAsync();
|
||||||
|
var (page, consoleMessages, networkRequests) = await _browserService.CreatePageWithDebugAsync(context, requestId);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
return await _scrapingService.ScrapeRadarScreenshotAsync(
|
||||||
|
suburb,
|
||||||
|
state,
|
||||||
|
debugFolder,
|
||||||
|
page,
|
||||||
|
consoleMessages,
|
||||||
|
networkRequests,
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Error during radar screenshot capture (RequestId: {RequestId})", requestId);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
await context.CloseAsync();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Error getting radar screenshot for suburb: {Suburb}, state: {State} (RequestId: {RequestId})", suburb, state, requestId);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
semaphore.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<LastUpdatedInfo?> GetLastUpdatedInfoAsync(string suburb, string state, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
// Return cached metadata if available, otherwise return null
|
||||||
|
var (_, metadata) = await _cacheService.GetCachedScreenshotWithMetadataAsync(suburb, state, cancellationToken);
|
||||||
|
|
||||||
|
if (metadata != null)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("Returning cached last updated info for {Suburb}, {State}", suburb, state);
|
||||||
|
return metadata;
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("No cached metadata found for {Suburb}, {State}", suburb, state);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task<string> GetCachedScreenshotPathAsync(string suburb, string state, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
return _cacheService.GetCachedScreenshotPathAsync(suburb, state, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task<bool> DeleteCachedLocationAsync(string suburb, string state, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
return _cacheService.DeleteCachedLocationAsync(suburb, state, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
_browserService?.Dispose();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,368 @@
|
|||||||
|
using BomLocalService.Services.Interfaces;
|
||||||
|
using Microsoft.Playwright;
|
||||||
|
|
||||||
|
namespace BomLocalService.Services;
|
||||||
|
|
||||||
|
public class BrowserService : IBrowserService
|
||||||
|
{
|
||||||
|
private readonly ILogger<BrowserService> _logger;
|
||||||
|
private readonly string _timezone;
|
||||||
|
private readonly IDebugService _debugService;
|
||||||
|
private IPlaywright? _playwright;
|
||||||
|
private IBrowser? _browser;
|
||||||
|
private readonly SemaphoreSlim _semaphore = new(1, 1);
|
||||||
|
|
||||||
|
public BrowserService(ILogger<BrowserService> logger, IConfiguration configuration, IDebugService debugService)
|
||||||
|
{
|
||||||
|
_logger = logger;
|
||||||
|
_timezone = configuration.GetValue<string>("Timezone") ?? "Australia/Brisbane";
|
||||||
|
_debugService = debugService;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or creates the Playwright instance
|
||||||
|
/// </summary>
|
||||||
|
private async Task<IPlaywright> GetPlaywrightAsync()
|
||||||
|
{
|
||||||
|
if (_playwright == null)
|
||||||
|
{
|
||||||
|
_playwright = await Microsoft.Playwright.Playwright.CreateAsync();
|
||||||
|
}
|
||||||
|
return _playwright;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or creates the browser instance
|
||||||
|
/// </summary>
|
||||||
|
public async Task<IBrowser> GetBrowserAsync()
|
||||||
|
{
|
||||||
|
if (_browser == null)
|
||||||
|
{
|
||||||
|
var playwright = await GetPlaywrightAsync();
|
||||||
|
_browser = await playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions
|
||||||
|
{
|
||||||
|
Headless = false, // Run in visible mode with virtual display (Xvfb)
|
||||||
|
Args = new[]
|
||||||
|
{
|
||||||
|
"--no-sandbox",
|
||||||
|
"--disable-setuid-sandbox",
|
||||||
|
"--disable-blink-features=AutomationControlled",
|
||||||
|
"--disable-dev-shm-usage",
|
||||||
|
"--disable-infobars",
|
||||||
|
"--disable-extensions",
|
||||||
|
"--disable-background-networking",
|
||||||
|
"--disable-background-timer-throttling",
|
||||||
|
"--disable-backgrounding-occluded-windows",
|
||||||
|
"--disable-breakpad",
|
||||||
|
"--disable-client-side-phishing-detection",
|
||||||
|
"--disable-component-extensions-with-background-pages",
|
||||||
|
"--disable-component-update",
|
||||||
|
"--disable-default-apps",
|
||||||
|
"--disable-domain-reliability",
|
||||||
|
"--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees",
|
||||||
|
"--disable-hang-monitor",
|
||||||
|
"--disable-ipc-flooding-protection",
|
||||||
|
"--disable-notifications",
|
||||||
|
"--disable-offer-store-unmasked-wallet-cards",
|
||||||
|
"--disable-popup-blocking",
|
||||||
|
"--disable-prompt-on-repost",
|
||||||
|
"--disable-renderer-backgrounding",
|
||||||
|
"--disable-sync",
|
||||||
|
"--force-color-profile=srgb",
|
||||||
|
"--metrics-recording-only",
|
||||||
|
"--no-first-run",
|
||||||
|
"--no-default-browser-check",
|
||||||
|
"--no-pings",
|
||||||
|
"--password-store=basic",
|
||||||
|
"--use-mock-keychain",
|
||||||
|
"--enable-automation=false",
|
||||||
|
"--exclude-switches=enable-automation",
|
||||||
|
"--disable-features=UserAgentClientHint",
|
||||||
|
"--enable-unsafe-swiftshader", // Required for software WebGL (Chrome 137+ deprecated automatic fallback)
|
||||||
|
"--use-gl=swiftshader" // Use SwiftShader for WebGL software rendering
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return _browser;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a new browser context with proper configuration
|
||||||
|
/// </summary>
|
||||||
|
public async Task<IBrowserContext> CreateContextAsync()
|
||||||
|
{
|
||||||
|
var browser = await GetBrowserAsync();
|
||||||
|
return await browser.NewContextAsync(new BrowserNewContextOptions
|
||||||
|
{
|
||||||
|
ViewportSize = new ViewportSize { Width = 1920, Height = 1080 },
|
||||||
|
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||||
|
Locale = "en-AU",
|
||||||
|
TimezoneId = _timezone,
|
||||||
|
Permissions = new[] { "geolocation" },
|
||||||
|
ExtraHTTPHeaders = new Dictionary<string, string>
|
||||||
|
{
|
||||||
|
{ "Accept-Language", "en-AU,en;q=0.9" },
|
||||||
|
{ "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8" },
|
||||||
|
{ "Accept-Encoding", "gzip, deflate, br" },
|
||||||
|
{ "Connection", "keep-alive" },
|
||||||
|
{ "Upgrade-Insecure-Requests", "1" },
|
||||||
|
{ "Sec-Fetch-Dest", "document" },
|
||||||
|
{ "Sec-Fetch-Mode", "navigate" },
|
||||||
|
{ "Sec-Fetch-Site", "none" },
|
||||||
|
{ "Cache-Control", "max-age=0" }
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a new page with anti-detection scripts and debug event handlers
|
||||||
|
/// </summary>
|
||||||
|
public async Task<(IPage page, List<(string type, string text, DateTime timestamp)> consoleMessages, List<(string method, string url, int? status, string resourceType, DateTime timestamp)> networkRequests)> CreatePageWithDebugAsync(
|
||||||
|
IBrowserContext context,
|
||||||
|
string requestId)
|
||||||
|
{
|
||||||
|
var page = await context.NewPageAsync();
|
||||||
|
|
||||||
|
// Set up console message and network request capture for debugging
|
||||||
|
var consoleMessages = new List<(string type, string text, DateTime timestamp)>();
|
||||||
|
var networkRequests = new List<(string method, string url, int? status, string resourceType, DateTime timestamp)>();
|
||||||
|
|
||||||
|
if (_debugService.IsEnabled)
|
||||||
|
{
|
||||||
|
// Capture page console messages (console.log, console.error, etc.)
|
||||||
|
page.Console += (sender, e) =>
|
||||||
|
{
|
||||||
|
consoleMessages.Add((e.Type.ToString(), e.Text, DateTime.UtcNow));
|
||||||
|
};
|
||||||
|
|
||||||
|
// Capture page errors (unhandled JavaScript errors)
|
||||||
|
page.PageError += (sender, e) =>
|
||||||
|
{
|
||||||
|
consoleMessages.Add(("error", $"PageError: {e}", DateTime.UtcNow));
|
||||||
|
};
|
||||||
|
|
||||||
|
// Capture context console messages (broader scope, may include more warnings)
|
||||||
|
context.Console += (sender, e) =>
|
||||||
|
{
|
||||||
|
consoleMessages.Add((e.Type.ToString(), $"[Context] {e.Text}", DateTime.UtcNow));
|
||||||
|
};
|
||||||
|
|
||||||
|
// Capture network requests
|
||||||
|
page.Request += (sender, e) =>
|
||||||
|
{
|
||||||
|
networkRequests.Add((e.Method, e.Url, null, e.ResourceType, DateTime.UtcNow));
|
||||||
|
};
|
||||||
|
|
||||||
|
// Capture failed requests (these often show up as console errors in browser)
|
||||||
|
page.RequestFailed += (sender, e) =>
|
||||||
|
{
|
||||||
|
networkRequests.Add((e.Method, e.Url, null, e.ResourceType, DateTime.UtcNow));
|
||||||
|
var failureText = e.Failure?.ToString() ?? "Unknown failure";
|
||||||
|
consoleMessages.Add(("error", $"Request failed: {e.Method} {e.Url} - {failureText}", DateTime.UtcNow));
|
||||||
|
};
|
||||||
|
|
||||||
|
page.Response += (sender, e) =>
|
||||||
|
{
|
||||||
|
// Update the request with response status
|
||||||
|
var matchingRequests = networkRequests.Where(r => r.url == e.Request.Url && r.status == null).ToList();
|
||||||
|
if (matchingRequests.Any())
|
||||||
|
{
|
||||||
|
var request = matchingRequests.Last();
|
||||||
|
var index = networkRequests.IndexOf(request);
|
||||||
|
networkRequests[index] = (request.method, request.url, e.Status, request.resourceType, request.timestamp);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Inject comprehensive anti-detection scripts (must run before page load)
|
||||||
|
await page.AddInitScriptAsync(GetAntiDetectionScript());
|
||||||
|
|
||||||
|
return (page, consoleMessages, networkRequests);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the anti-detection JavaScript script
|
||||||
|
/// </summary>
|
||||||
|
private static string GetAntiDetectionScript()
|
||||||
|
{
|
||||||
|
return @"
|
||||||
|
(function() {
|
||||||
|
'use strict';
|
||||||
|
|
||||||
|
// Remove webdriver property completely
|
||||||
|
Object.defineProperty(navigator, 'webdriver', {
|
||||||
|
get: () => false,
|
||||||
|
configurable: true
|
||||||
|
});
|
||||||
|
|
||||||
|
// Delete it from prototype chain
|
||||||
|
try {
|
||||||
|
delete navigator.__proto__.webdriver;
|
||||||
|
} catch (e) {}
|
||||||
|
|
||||||
|
// Override chrome object
|
||||||
|
window.chrome = {
|
||||||
|
runtime: {},
|
||||||
|
loadTimes: function() {},
|
||||||
|
csi: function() {},
|
||||||
|
app: {}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Override plugins with realistic data
|
||||||
|
Object.defineProperty(navigator, 'plugins', {
|
||||||
|
get: () => {
|
||||||
|
const plugins = [];
|
||||||
|
for (let i = 0; i < 3; i++) {
|
||||||
|
plugins.push({
|
||||||
|
0: { type: 'application/x-google-chrome-pdf', suffixes: 'pdf', description: 'Portable Document Format' },
|
||||||
|
description: 'Portable Document Format',
|
||||||
|
filename: 'internal-pdf-viewer',
|
||||||
|
length: 1,
|
||||||
|
name: 'Chrome PDF Plugin'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return plugins;
|
||||||
|
},
|
||||||
|
configurable: true
|
||||||
|
});
|
||||||
|
|
||||||
|
// Override languages
|
||||||
|
Object.defineProperty(navigator, 'languages', {
|
||||||
|
get: () => ['en-AU', 'en', 'en-US'],
|
||||||
|
configurable: true
|
||||||
|
});
|
||||||
|
|
||||||
|
// Override language
|
||||||
|
Object.defineProperty(navigator, 'language', {
|
||||||
|
get: () => 'en-AU',
|
||||||
|
configurable: true
|
||||||
|
});
|
||||||
|
|
||||||
|
// Override vendor
|
||||||
|
Object.defineProperty(navigator, 'vendor', {
|
||||||
|
get: () => 'Google Inc.',
|
||||||
|
configurable: true
|
||||||
|
});
|
||||||
|
|
||||||
|
// Override platform
|
||||||
|
Object.defineProperty(navigator, 'platform', {
|
||||||
|
get: () => 'Win32',
|
||||||
|
configurable: true
|
||||||
|
});
|
||||||
|
|
||||||
|
// Override hardwareConcurrency
|
||||||
|
Object.defineProperty(navigator, 'hardwareConcurrency', {
|
||||||
|
get: () => 8,
|
||||||
|
configurable: true
|
||||||
|
});
|
||||||
|
|
||||||
|
// Override deviceMemory
|
||||||
|
if ('deviceMemory' in navigator) {
|
||||||
|
Object.defineProperty(navigator, 'deviceMemory', {
|
||||||
|
get: () => 8,
|
||||||
|
configurable: true
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Spoof connection - use named functions to avoid warnings
|
||||||
|
Object.defineProperty(navigator, 'connection', {
|
||||||
|
get: () => ({
|
||||||
|
effectiveType: '4g',
|
||||||
|
rtt: 50,
|
||||||
|
downlink: 10,
|
||||||
|
saveData: false,
|
||||||
|
onchange: null,
|
||||||
|
addEventListener: function addEventListener() {},
|
||||||
|
removeEventListener: function removeEventListener() {},
|
||||||
|
dispatchEvent: function dispatchEvent() { return true; }
|
||||||
|
}),
|
||||||
|
configurable: true
|
||||||
|
});
|
||||||
|
|
||||||
|
// Override permissions API - bind context to avoid illegal invocation
|
||||||
|
const originalQuery = window.navigator.permissions.query.bind(window.navigator.permissions);
|
||||||
|
window.navigator.permissions.query = function query(parameters) {
|
||||||
|
return parameters.name === 'notifications' ?
|
||||||
|
Promise.resolve({ state: Notification.permission }) :
|
||||||
|
originalQuery(parameters);
|
||||||
|
};
|
||||||
|
|
||||||
|
// Override getBattery - bind context to avoid illegal invocation
|
||||||
|
if ('getBattery' in navigator) {
|
||||||
|
navigator.getBattery = function getBattery() {
|
||||||
|
return Promise.resolve({
|
||||||
|
charging: true,
|
||||||
|
chargingTime: 0,
|
||||||
|
dischargingTime: Infinity,
|
||||||
|
level: 1,
|
||||||
|
onchargingchange: null,
|
||||||
|
onchargingtimechange: null,
|
||||||
|
ondischargingtimechange: null,
|
||||||
|
onlevelchange: null,
|
||||||
|
addEventListener: function addEventListener() {},
|
||||||
|
removeEventListener: function removeEventListener() {},
|
||||||
|
dispatchEvent: function dispatchEvent() { return true; }
|
||||||
|
});
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// DON'T use Proxy on navigator - it breaks APIs like getGamepads()
|
||||||
|
// Just hide webdriver property directly
|
||||||
|
try {
|
||||||
|
delete navigator.__proto__.webdriver;
|
||||||
|
} catch (e) {}
|
||||||
|
|
||||||
|
// AudioContext fingerprinting
|
||||||
|
if (window.AudioContext) {
|
||||||
|
const originalAudioContext = window.AudioContext;
|
||||||
|
window.AudioContext = function() {
|
||||||
|
const ctx = new originalAudioContext();
|
||||||
|
const originalCreateOscillator = ctx.createOscillator;
|
||||||
|
ctx.createOscillator = function() {
|
||||||
|
const oscillator = originalCreateOscillator.call(this);
|
||||||
|
const originalFrequency = Object.getOwnPropertyDescriptor(
|
||||||
|
AudioParam.prototype, 'value'
|
||||||
|
).get;
|
||||||
|
Object.defineProperty(oscillator.frequency, 'value', {
|
||||||
|
get: originalFrequency,
|
||||||
|
set: function() {}
|
||||||
|
});
|
||||||
|
return oscillator;
|
||||||
|
};
|
||||||
|
return ctx;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
";
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Initializes the browser (pre-warming)
|
||||||
|
/// </summary>
|
||||||
|
public async Task InitializeAsync(CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
_logger.LogInformation("Pre-warming browser");
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await GetBrowserAsync();
|
||||||
|
_logger.LogInformation("Browser initialized and ready");
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(ex, "Failed to pre-warm browser, will initialize on first use");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the semaphore for synchronizing browser operations
|
||||||
|
/// </summary>
|
||||||
|
public SemaphoreSlim GetSemaphore() => _semaphore;
|
||||||
|
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
_browser?.DisposeAsync().AsTask().Wait();
|
||||||
|
_playwright?.Dispose();
|
||||||
|
_semaphore?.Dispose();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,141 @@
|
|||||||
|
using BomLocalService.Utilities;
|
||||||
|
using Microsoft.Extensions.Hosting;
|
||||||
|
|
||||||
|
namespace BomLocalService.Services;
|
||||||
|
|
||||||
|
public class CacheCleanupService : BackgroundService
|
||||||
|
{
|
||||||
|
private readonly ILogger<CacheCleanupService> _logger;
|
||||||
|
private readonly string _cacheDirectory;
|
||||||
|
private readonly int _retentionHours;
|
||||||
|
private readonly TimeSpan _cleanupInterval;
|
||||||
|
|
||||||
|
public CacheCleanupService(ILogger<CacheCleanupService> logger, IConfiguration configuration)
|
||||||
|
{
|
||||||
|
_logger = logger;
|
||||||
|
_cacheDirectory = FilePathHelper.GetCacheDirectory(configuration);
|
||||||
|
_retentionHours = configuration.GetValue<int>("CacheRetentionHours", 24);
|
||||||
|
var cleanupIntervalHours = configuration.GetValue<int>("CacheCleanup:IntervalHours", 1);
|
||||||
|
_cleanupInterval = TimeSpan.FromHours(cleanupIntervalHours);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
|
{
|
||||||
|
_logger.LogInformation("Cache cleanup service started. Retention: {RetentionHours} hours, Interval: {Interval}",
|
||||||
|
_retentionHours, _cleanupInterval);
|
||||||
|
|
||||||
|
while (!stoppingToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await CleanupOldCacheFilesAsync(stoppingToken);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Error during cache cleanup");
|
||||||
|
}
|
||||||
|
|
||||||
|
await Task.Delay(_cleanupInterval, stoppingToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Task CleanupOldCacheFilesAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
if (!Directory.Exists(_cacheDirectory))
|
||||||
|
{
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
|
var cutoffTime = DateTime.UtcNow.AddHours(-_retentionHours);
|
||||||
|
var deletedCount = 0;
|
||||||
|
var totalSize = 0L;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
// Get all PNG files in cache directory
|
||||||
|
var pngFiles = Directory.GetFiles(_cacheDirectory, "*.png", SearchOption.TopDirectoryOnly);
|
||||||
|
|
||||||
|
foreach (var file in pngFiles)
|
||||||
|
{
|
||||||
|
if (cancellationToken.IsCancellationRequested)
|
||||||
|
break;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var fileInfo = new FileInfo(file);
|
||||||
|
var fileTime = fileInfo.LastWriteTimeUtc;
|
||||||
|
|
||||||
|
// Delete if file is older than retention period
|
||||||
|
if (fileTime < cutoffTime)
|
||||||
|
{
|
||||||
|
var fileSize = fileInfo.Length;
|
||||||
|
File.Delete(file);
|
||||||
|
deletedCount++;
|
||||||
|
totalSize += fileSize;
|
||||||
|
|
||||||
|
// Also delete associated metadata file if it exists
|
||||||
|
var metadataFile = FilePathHelper.GetMetadataFilePath(file);
|
||||||
|
if (File.Exists(metadataFile))
|
||||||
|
{
|
||||||
|
File.Delete(metadataFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("Deleted old cache file: {File} (age: {Age})",
|
||||||
|
Path.GetFileName(file), DateTime.UtcNow - fileTime);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(ex, "Failed to delete cache file: {File}", file);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (deletedCount > 0)
|
||||||
|
{
|
||||||
|
var sizeInMB = totalSize / (1024.0 * 1024.0);
|
||||||
|
_logger.LogInformation("Cache cleanup completed. Deleted {Count} files ({Size:F2} MB)",
|
||||||
|
deletedCount, sizeInMB);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Error during cache file enumeration");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also clean up old debug directories
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var debugDir = Path.Combine(_cacheDirectory, "debug");
|
||||||
|
if (Directory.Exists(debugDir))
|
||||||
|
{
|
||||||
|
var debugDirs = Directory.GetDirectories(debugDir);
|
||||||
|
foreach (var dir in debugDirs)
|
||||||
|
{
|
||||||
|
if (cancellationToken.IsCancellationRequested)
|
||||||
|
break;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var dirInfo = new DirectoryInfo(dir);
|
||||||
|
if (dirInfo.LastWriteTimeUtc < cutoffTime)
|
||||||
|
{
|
||||||
|
Directory.Delete(dir, recursive: true);
|
||||||
|
_logger.LogDebug("Deleted old debug directory: {Dir}", dir);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(ex, "Failed to delete debug directory: {Dir}", dir);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(ex, "Error cleaning up debug directories: {Error}", ex.Message);
|
||||||
|
}
|
||||||
|
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,210 @@
|
|||||||
|
using BomLocalService.Services.Interfaces;
|
||||||
|
using BomLocalService.Utilities;
|
||||||
|
using Microsoft.Extensions.Hosting;
|
||||||
|
|
||||||
|
namespace BomLocalService.Services;
|
||||||
|
|
||||||
|
public class CacheManagementService : BackgroundService
|
||||||
|
{
|
||||||
|
private readonly ILogger<CacheManagementService> _logger;
|
||||||
|
private readonly IBomRadarService _bomRadarService;
|
||||||
|
private readonly IBrowserService _browserService;
|
||||||
|
private readonly IConfiguration _configuration;
|
||||||
|
private readonly TimeSpan _checkInterval;
|
||||||
|
private readonly HashSet<string> _activeUpdates = new();
|
||||||
|
private readonly object _lock = new();
|
||||||
|
|
||||||
|
public CacheManagementService(
|
||||||
|
ILogger<CacheManagementService> logger,
|
||||||
|
IBomRadarService bomRadarService,
|
||||||
|
IBrowserService browserService,
|
||||||
|
IConfiguration configuration)
|
||||||
|
{
|
||||||
|
_logger = logger;
|
||||||
|
_bomRadarService = bomRadarService;
|
||||||
|
_browserService = browserService;
|
||||||
|
_configuration = configuration;
|
||||||
|
var checkIntervalMinutes = configuration.GetValue<int>("CacheManagement:CheckIntervalMinutes", 5);
|
||||||
|
_checkInterval = TimeSpan.FromMinutes(checkIntervalMinutes);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
|
{
|
||||||
|
_logger.LogInformation("Cache management service started. Check interval: {Interval}", _checkInterval);
|
||||||
|
|
||||||
|
// Wait a bit for the service to fully initialize
|
||||||
|
var initialDelaySeconds = _configuration.GetValue<int>("CacheManagement:InitialDelaySeconds", 10);
|
||||||
|
await Task.Delay(TimeSpan.FromSeconds(initialDelaySeconds), stoppingToken);
|
||||||
|
|
||||||
|
// Pre-warm the browser before starting cache updates
|
||||||
|
_logger.LogInformation("Pre-warming browser before cache updates");
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await _browserService.InitializeAsync(stoppingToken);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(ex, "Failed to pre-warm browser, will continue anyway");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get initial list of locations to manage (could be from config or discovered from cache)
|
||||||
|
var locationsToManage = GetLocationsToManage();
|
||||||
|
|
||||||
|
// Initial cache update on startup
|
||||||
|
_logger.LogInformation("Performing initial cache update for {Count} locations", locationsToManage.Count);
|
||||||
|
int initialUpdatesTriggered = 0;
|
||||||
|
int initialCachesValid = 0;
|
||||||
|
|
||||||
|
foreach (var (suburb, state) in locationsToManage)
|
||||||
|
{
|
||||||
|
if (stoppingToken.IsCancellationRequested)
|
||||||
|
break;
|
||||||
|
|
||||||
|
var (triggered, isValid) = await UpdateCacheIfNeededAsync(suburb, state, stoppingToken);
|
||||||
|
if (triggered)
|
||||||
|
{
|
||||||
|
initialUpdatesTriggered++;
|
||||||
|
}
|
||||||
|
else if (isValid)
|
||||||
|
{
|
||||||
|
initialCachesValid++;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stagger updates to avoid overwhelming the system
|
||||||
|
var updateStaggerSeconds = _configuration.GetValue<int>("CacheManagement:UpdateStaggerSeconds", 2);
|
||||||
|
await Task.Delay(TimeSpan.FromSeconds(updateStaggerSeconds), stoppingToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogInformation("Initial cache update completed: {UpdatesTriggered} updates triggered, {CachesValid} caches valid",
|
||||||
|
initialUpdatesTriggered, initialCachesValid);
|
||||||
|
|
||||||
|
// Periodic cache updates
|
||||||
|
while (!stoppingToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await Task.Delay(_checkInterval, stoppingToken);
|
||||||
|
|
||||||
|
// Refresh list of locations (in case new ones were cached)
|
||||||
|
locationsToManage = GetLocationsToManage();
|
||||||
|
_logger.LogInformation("Starting periodic cache check for {Count} locations", locationsToManage.Count);
|
||||||
|
|
||||||
|
int updatesTriggered = 0;
|
||||||
|
int cachesValid = 0;
|
||||||
|
|
||||||
|
foreach (var (suburb, state) in locationsToManage)
|
||||||
|
{
|
||||||
|
if (stoppingToken.IsCancellationRequested)
|
||||||
|
break;
|
||||||
|
|
||||||
|
var (triggered, isValid) = await UpdateCacheIfNeededAsync(suburb, state, stoppingToken);
|
||||||
|
if (triggered)
|
||||||
|
{
|
||||||
|
updatesTriggered++;
|
||||||
|
}
|
||||||
|
else if (isValid)
|
||||||
|
{
|
||||||
|
cachesValid++;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Small delay between locations
|
||||||
|
var locationStaggerSeconds = _configuration.GetValue<int>("CacheManagement:LocationStaggerSeconds", 1);
|
||||||
|
await Task.Delay(TimeSpan.FromSeconds(locationStaggerSeconds), stoppingToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogInformation("Periodic cache check completed: {UpdatesTriggered} updates triggered, {CachesValid} caches valid",
|
||||||
|
updatesTriggered, cachesValid);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Error during cache management cycle");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<(bool updateTriggered, bool isValid)> UpdateCacheIfNeededAsync(string suburb, string state, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var locationKey = LocationHelper.GetLocationKey(suburb, state);
|
||||||
|
|
||||||
|
// Check if update is already in progress
|
||||||
|
lock (_lock)
|
||||||
|
{
|
||||||
|
if (_activeUpdates.Contains(locationKey))
|
||||||
|
{
|
||||||
|
_logger.LogInformation("Update already in progress for {Suburb}, {State}, skipping", suburb, state);
|
||||||
|
return (false, false);
|
||||||
|
}
|
||||||
|
_activeUpdates.Add(locationKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var status = await _bomRadarService.TriggerCacheUpdateAsync(suburb, state, cancellationToken);
|
||||||
|
|
||||||
|
if (status.UpdateTriggered)
|
||||||
|
{
|
||||||
|
_logger.LogInformation("Cache update triggered for {Suburb}, {State} (expired at {ExpiresAt})",
|
||||||
|
suburb, state, status.CacheExpiresAt);
|
||||||
|
return (true, false);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_logger.LogInformation("Cache is valid for {Suburb}, {State} (expires at {ExpiresAt})",
|
||||||
|
suburb, state, status.CacheExpiresAt);
|
||||||
|
return (false, true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Error checking/updating cache for {Suburb}, {State}", suburb, state);
|
||||||
|
return (false, false);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
lock (_lock)
|
||||||
|
{
|
||||||
|
_activeUpdates.Remove(locationKey);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<(string suburb, string state)> GetLocationsToManage()
|
||||||
|
{
|
||||||
|
var locations = new List<(string, string)>();
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
// Get locations from existing cache files
|
||||||
|
var cacheDirectory = FilePathHelper.GetCacheDirectory(_configuration);
|
||||||
|
if (Directory.Exists(cacheDirectory))
|
||||||
|
{
|
||||||
|
var pngFiles = Directory.GetFiles(cacheDirectory, "*.png", SearchOption.TopDirectoryOnly);
|
||||||
|
var seen = new HashSet<string>();
|
||||||
|
|
||||||
|
foreach (var file in pngFiles)
|
||||||
|
{
|
||||||
|
var location = LocationHelper.ParseLocationFromFilename(file);
|
||||||
|
if (location.HasValue)
|
||||||
|
{
|
||||||
|
var locationKey = LocationHelper.GetLocationKey(location.Value.suburb, location.Value.state);
|
||||||
|
if (!seen.Contains(locationKey))
|
||||||
|
{
|
||||||
|
seen.Add(locationKey);
|
||||||
|
locations.Add(location.Value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Could also read from config if you want to pre-configure locations
|
||||||
|
// var configuredLocations = _configuration.GetSection("ManagedLocations").Get<List<LocationConfig>>();
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(ex, "Error discovering locations from cache");
|
||||||
|
}
|
||||||
|
|
||||||
|
return locations;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,189 @@
|
|||||||
|
using BomLocalService.Models;
|
||||||
|
using BomLocalService.Services.Interfaces;
|
||||||
|
using BomLocalService.Utilities;
|
||||||
|
using System.Text.Json;
|
||||||
|
|
||||||
|
namespace BomLocalService.Services;
|
||||||
|
|
||||||
|
public class CacheService : ICacheService
|
||||||
|
{
|
||||||
|
private readonly ILogger<CacheService> _logger;
|
||||||
|
private readonly string _cacheDirectory;
|
||||||
|
private readonly double _cacheExpirationMinutes;
|
||||||
|
|
||||||
|
public CacheService(ILogger<CacheService> logger, IConfiguration configuration)
|
||||||
|
{
|
||||||
|
_logger = logger;
|
||||||
|
_cacheDirectory = FilePathHelper.GetCacheDirectory(configuration);
|
||||||
|
_cacheExpirationMinutes = configuration.GetValue<double>("CacheExpirationMinutes", 15.5);
|
||||||
|
|
||||||
|
// Ensure cache directory exists
|
||||||
|
Directory.CreateDirectory(_cacheDirectory);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the cached screenshot path and metadata for a location
|
||||||
|
/// </summary>
|
||||||
|
public async Task<(string? screenshotPath, LastUpdatedInfo? metadata)> GetCachedScreenshotWithMetadataAsync(
|
||||||
|
string suburb,
|
||||||
|
string state,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
var pattern = FilePathHelper.GetCacheFilePattern(suburb, state);
|
||||||
|
_logger.LogDebug("Looking for cached files matching pattern: {Pattern} in directory: {Directory}", pattern, _cacheDirectory);
|
||||||
|
|
||||||
|
var files = Directory.GetFiles(_cacheDirectory, pattern)
|
||||||
|
.OrderByDescending(f =>
|
||||||
|
{
|
||||||
|
// Try to extract timestamp from filename first
|
||||||
|
var timestamp = LocationHelper.ParseTimestampFromFilename(f);
|
||||||
|
if (timestamp.HasValue)
|
||||||
|
{
|
||||||
|
return timestamp.Value;
|
||||||
|
}
|
||||||
|
// Fallback to file write time
|
||||||
|
return File.GetLastWriteTime(f);
|
||||||
|
})
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
var screenshotPath = files.FirstOrDefault();
|
||||||
|
if (string.IsNullOrEmpty(screenshotPath) || !File.Exists(screenshotPath))
|
||||||
|
{
|
||||||
|
return (null, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
var metadata = await LoadMetadataAsync(screenshotPath, cancellationToken);
|
||||||
|
return (screenshotPath, metadata);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Saves metadata alongside a screenshot
|
||||||
|
/// </summary>
|
||||||
|
public async Task SaveMetadataAsync(string screenshotPath, LastUpdatedInfo metadata, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var metadataPath = FilePathHelper.GetMetadataFilePath(screenshotPath);
|
||||||
|
var json = JsonSerializer.Serialize(metadata, new JsonSerializerOptions { WriteIndented = true });
|
||||||
|
// Use CancellationToken.None for file writes to avoid cancellation issues
|
||||||
|
// The screenshot is already saved, metadata save is best-effort
|
||||||
|
await File.WriteAllTextAsync(metadataPath, json, CancellationToken.None);
|
||||||
|
_logger.LogDebug("Saved metadata to: {Path}", metadataPath);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
// Log but don't fail - screenshot is already saved
|
||||||
|
_logger.LogWarning(ex, "Failed to save metadata for screenshot: {Path}", screenshotPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Loads metadata for a screenshot
|
||||||
|
/// </summary>
|
||||||
|
public async Task<LastUpdatedInfo?> LoadMetadataAsync(string screenshotPath, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
var metadataPath = FilePathHelper.GetMetadataFilePath(screenshotPath);
|
||||||
|
if (!File.Exists(metadataPath))
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var json = await File.ReadAllTextAsync(metadataPath, cancellationToken);
|
||||||
|
return JsonSerializer.Deserialize<LastUpdatedInfo>(json);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(ex, "Failed to load metadata from: {Path}", metadataPath);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Checks if cache is still valid based on observation time
|
||||||
|
/// </summary>
|
||||||
|
public bool IsCacheValid(LastUpdatedInfo? metadata)
|
||||||
|
{
|
||||||
|
if (metadata == null)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// BOM updates observations at :00, :15, :30, :45 (every 15 minutes by default)
|
||||||
|
// Cache expiration includes buffer (e.g., 15.5 minutes = 15 min + 30 sec buffer)
|
||||||
|
var nextUpdateTime = metadata.ObservationTime.AddMinutes(_cacheExpirationMinutes);
|
||||||
|
return DateTime.UtcNow < nextUpdateTime;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the path to the cached screenshot for a location (simple version, ordered by creation time)
|
||||||
|
/// </summary>
|
||||||
|
public Task<string> GetCachedScreenshotPathAsync(string suburb, string state, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
var pattern = FilePathHelper.GetCacheFilePattern(suburb, state);
|
||||||
|
var files = Directory.GetFiles(_cacheDirectory, pattern)
|
||||||
|
.OrderByDescending(f => File.GetCreationTime(f))
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
return Task.FromResult(files.FirstOrDefault() ?? string.Empty);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Deletes all cached files for a location
|
||||||
|
/// </summary>
|
||||||
|
public Task<bool> DeleteCachedLocationAsync(string suburb, string state, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
var pattern = FilePathHelper.GetCacheFilePattern(suburb, state);
|
||||||
|
var deleted = false;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
// Delete all PNG files for this location
|
||||||
|
var pngFiles = Directory.GetFiles(_cacheDirectory, pattern);
|
||||||
|
foreach (var pngFile in pngFiles)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
File.Delete(pngFile);
|
||||||
|
_logger.LogInformation("Deleted cached screenshot: {File}", pngFile);
|
||||||
|
deleted = true;
|
||||||
|
|
||||||
|
// Also delete associated metadata JSON file
|
||||||
|
var metadataFile = FilePathHelper.GetMetadataFilePath(pngFile);
|
||||||
|
if (File.Exists(metadataFile))
|
||||||
|
{
|
||||||
|
File.Delete(metadataFile);
|
||||||
|
_logger.LogInformation("Deleted metadata file: {File}", metadataFile);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(ex, "Failed to delete file: {File}", pngFile);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (deleted)
|
||||||
|
{
|
||||||
|
_logger.LogInformation("Deleted all cached files for location: {Suburb}, {State}", suburb, state);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_logger.LogDebug("No cached files found to delete for location: {Suburb}, {State}", suburb, state);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Error deleting cached location: {Suburb}, {State}", suburb, state);
|
||||||
|
return Task.FromResult(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
return Task.FromResult(deleted);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the cache directory path
|
||||||
|
/// </summary>
|
||||||
|
public string GetCacheDirectory() => _cacheDirectory;
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,176 @@
|
|||||||
|
using BomLocalService.Services.Interfaces;
|
||||||
|
using BomLocalService.Utilities;
|
||||||
|
using Microsoft.Playwright;
|
||||||
|
|
||||||
|
namespace BomLocalService.Services;
|
||||||
|
|
||||||
|
public class DebugService : IDebugService
|
||||||
|
{
|
||||||
|
private readonly ILogger<DebugService> _logger;
|
||||||
|
private readonly bool _enabled;
|
||||||
|
private readonly string _debugDirectory;
|
||||||
|
private readonly int _waitMs;
|
||||||
|
|
||||||
|
public DebugService(ILogger<DebugService> logger, IConfiguration configuration)
|
||||||
|
{
|
||||||
|
_logger = logger;
|
||||||
|
_waitMs = configuration.GetValue<int>("Debug:WaitMs", 2000);
|
||||||
|
|
||||||
|
// Check all possible ways the config might be set
|
||||||
|
var debugEnabledEnv = Environment.GetEnvironmentVariable("DEBUG__ENABLED");
|
||||||
|
var debugEnabledConfig = configuration.GetValue<bool>("Debug:Enabled", false);
|
||||||
|
var debugEnabledFromConfig = configuration["Debug:Enabled"];
|
||||||
|
|
||||||
|
_logger.LogInformation("Debug configuration check - Env var DEBUG__ENABLED: {EnvVar}, Config Debug:Enabled: {ConfigValue}, Config string: {ConfigString}",
|
||||||
|
debugEnabledEnv ?? "null", debugEnabledConfig, debugEnabledFromConfig ?? "null");
|
||||||
|
|
||||||
|
// Try environment variable first, then config
|
||||||
|
_enabled = !string.IsNullOrEmpty(debugEnabledEnv)
|
||||||
|
? bool.TryParse(debugEnabledEnv, out var envBool) && envBool
|
||||||
|
: debugEnabledConfig;
|
||||||
|
|
||||||
|
var cacheDirectory = FilePathHelper.GetCacheDirectory(configuration);
|
||||||
|
_debugDirectory = Path.Combine(cacheDirectory, "debug");
|
||||||
|
|
||||||
|
if (_enabled)
|
||||||
|
{
|
||||||
|
Directory.CreateDirectory(_debugDirectory);
|
||||||
|
_logger.LogInformation("Debug mode is ENABLED. Debug files will be saved to: {DebugDirectory}", _debugDirectory);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_logger.LogInformation("Debug mode is DISABLED");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool IsEnabled => _enabled;
|
||||||
|
|
||||||
|
public string CreateRequestFolder(string requestId)
|
||||||
|
{
|
||||||
|
if (!_enabled) return string.Empty;
|
||||||
|
|
||||||
|
var requestFolder = Path.Combine(_debugDirectory, requestId);
|
||||||
|
Directory.CreateDirectory(requestFolder);
|
||||||
|
_logger.LogDebug("Created debug folder for request: {RequestId} at {Path}", requestId, requestFolder);
|
||||||
|
return requestFolder;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task SaveStepDebugAsync(string requestFolder, int stepNumber, string stepName, IPage page, List<(string type, string text, DateTime timestamp)>? consoleMessages = null, List<(string method, string url, int? status, string resourceType, DateTime timestamp)>? networkRequests = null, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
if (!_enabled || string.IsNullOrEmpty(requestFolder)) return;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
// Wait for page to be fully loaded before taking screenshot
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await page.WaitForLoadStateAsync(LoadState.NetworkIdle, new PageWaitForLoadStateOptions { Timeout = 10000 });
|
||||||
|
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 5000 });
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// If network idle times out, at least wait for DOM
|
||||||
|
}
|
||||||
|
await page.WaitForTimeoutAsync(_waitMs); // Additional wait for dynamic content and animations
|
||||||
|
|
||||||
|
var stepFolder = Path.Combine(requestFolder, $"step_{stepNumber:D2}_{stepName}");
|
||||||
|
Directory.CreateDirectory(stepFolder);
|
||||||
|
|
||||||
|
// Save screenshot
|
||||||
|
var screenshotPath = Path.Combine(stepFolder, "screenshot.png");
|
||||||
|
await page.ScreenshotAsync(new PageScreenshotOptions
|
||||||
|
{
|
||||||
|
Path = screenshotPath,
|
||||||
|
FullPage = true
|
||||||
|
});
|
||||||
|
|
||||||
|
// Save HTML
|
||||||
|
var htmlContent = await page.ContentAsync();
|
||||||
|
var htmlPath = Path.Combine(stepFolder, "page.html");
|
||||||
|
await File.WriteAllTextAsync(htmlPath, htmlContent, cancellationToken);
|
||||||
|
|
||||||
|
// Save console messages if provided
|
||||||
|
if (consoleMessages != null && consoleMessages.Count > 0)
|
||||||
|
{
|
||||||
|
var consolePath = Path.Combine(stepFolder, "console.log");
|
||||||
|
var consoleText = string.Join("\n", consoleMessages.Select(msg => $"[{msg.timestamp:HH:mm:ss.fff}] [{msg.type}] {msg.text}"));
|
||||||
|
await File.WriteAllTextAsync(consolePath, consoleText, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save network requests summary if provided
|
||||||
|
if (networkRequests != null && networkRequests.Count > 0)
|
||||||
|
{
|
||||||
|
var networkPath = Path.Combine(stepFolder, "network.log");
|
||||||
|
var networkText = string.Join("\n", networkRequests.Select(req =>
|
||||||
|
$"[{req.timestamp:HH:mm:ss.fff}] {req.method} {req.url} -> {req.status?.ToString() ?? "pending"} ({req.resourceType})"
|
||||||
|
));
|
||||||
|
await File.WriteAllTextAsync(networkPath, networkText, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("Saved debug files for step {StepNumber} ({StepName}) to {Path}", stepNumber, stepName, stepFolder);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(ex, "Failed to save debug files for step {StepNumber} ({StepName})", stepNumber, stepName);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task SaveErrorDebugAsync(string requestFolder, string errorMessage, IPage? page = null,
|
||||||
|
List<(string type, string text, DateTime timestamp)>? consoleMessages = null,
|
||||||
|
List<(string method, string url, int? status, string resourceType, DateTime timestamp)>? networkRequests = null,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
if (!_enabled || string.IsNullOrEmpty(requestFolder)) return;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var errorFolder = Path.Combine(requestFolder, "error");
|
||||||
|
Directory.CreateDirectory(errorFolder);
|
||||||
|
|
||||||
|
// Save error message
|
||||||
|
var errorPath = Path.Combine(errorFolder, "error.txt");
|
||||||
|
await File.WriteAllTextAsync(errorPath, errorMessage, cancellationToken);
|
||||||
|
|
||||||
|
// Save screenshot if page is available
|
||||||
|
if (page != null)
|
||||||
|
{
|
||||||
|
var screenshotPath = Path.Combine(errorFolder, "screenshot.png");
|
||||||
|
await page.ScreenshotAsync(new PageScreenshotOptions
|
||||||
|
{
|
||||||
|
Path = screenshotPath,
|
||||||
|
FullPage = true
|
||||||
|
});
|
||||||
|
|
||||||
|
// Save HTML
|
||||||
|
var htmlContent = await page.ContentAsync();
|
||||||
|
var htmlPath = Path.Combine(errorFolder, "page.html");
|
||||||
|
await File.WriteAllTextAsync(htmlPath, htmlContent, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save console messages if provided
|
||||||
|
if (consoleMessages != null && consoleMessages.Count > 0)
|
||||||
|
{
|
||||||
|
var consolePath = Path.Combine(errorFolder, "console.log");
|
||||||
|
var consoleText = string.Join("\n", consoleMessages.Select(msg => $"[{msg.timestamp:HH:mm:ss.fff}] [{msg.type}] {msg.text}"));
|
||||||
|
await File.WriteAllTextAsync(consolePath, consoleText, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save network requests summary if provided
|
||||||
|
if (networkRequests != null && networkRequests.Count > 0)
|
||||||
|
{
|
||||||
|
var networkPath = Path.Combine(errorFolder, "network.log");
|
||||||
|
var networkText = string.Join("\n", networkRequests.Select(req =>
|
||||||
|
$"[{req.timestamp:HH:mm:ss.fff}] {req.method} {req.url} -> {req.status?.ToString() ?? "pending"} ({req.resourceType})"
|
||||||
|
));
|
||||||
|
await File.WriteAllTextAsync(networkPath, networkText, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("Saved error debug files to {Path}", errorFolder);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(ex, "Failed to save error debug files");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
using BomLocalService.Models;
|
||||||
|
|
||||||
|
namespace BomLocalService.Services.Interfaces;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Main service interface for BOM radar screenshot operations.
|
||||||
|
/// Orchestrates cache management, browser automation, and web scraping to provide radar screenshots for Australian locations.
|
||||||
|
/// </summary>
|
||||||
|
public interface IBomRadarService
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Gets a cached radar screenshot for a location.
|
||||||
|
/// Returns the screenshot response if available in cache, otherwise returns null.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
|
||||||
|
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
|
||||||
|
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
|
||||||
|
/// <returns>Radar screenshot response with image path and metadata, or null if not cached</returns>
|
||||||
|
Task<RadarScreenshotResponse?> GetCachedScreenshotAsync(string suburb, string state, CancellationToken cancellationToken = default);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Triggers a cache update for a location.
|
||||||
|
/// If cache is missing or expired, initiates a background update and returns status information.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
|
||||||
|
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
|
||||||
|
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
|
||||||
|
/// <returns>Status information about the cache update operation</returns>
|
||||||
|
Task<CacheUpdateStatus> TriggerCacheUpdateAsync(string suburb, string state, CancellationToken cancellationToken = default);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets metadata about the cached radar data for a location.
|
||||||
|
/// Returns observation time, forecast time, weather station, and distance information.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
|
||||||
|
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
|
||||||
|
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
|
||||||
|
/// <returns>Last updated information, or null if no cached data exists</returns>
|
||||||
|
Task<LastUpdatedInfo?> GetLastUpdatedInfoAsync(string suburb, string state, CancellationToken cancellationToken = default);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the file system path to the cached screenshot for a location.
|
||||||
|
/// Returns empty string if no cached screenshot exists.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
|
||||||
|
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
|
||||||
|
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
|
||||||
|
/// <returns>File system path to the cached screenshot, or empty string if not found</returns>
|
||||||
|
Task<string> GetCachedScreenshotPathAsync(string suburb, string state, CancellationToken cancellationToken = default);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Deletes all cached files (screenshot and metadata) for a location.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
|
||||||
|
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
|
||||||
|
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
|
||||||
|
/// <returns>True if files were deleted, false if no cached files existed</returns>
|
||||||
|
Task<bool> DeleteCachedLocationAsync(string suburb, string state, CancellationToken cancellationToken = default);
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
using Microsoft.Playwright;
|
||||||
|
|
||||||
|
namespace BomLocalService.Services.Interfaces;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Service interface for managing Playwright browser instances and automation.
|
||||||
|
/// Handles browser lifecycle, context creation, and anti-detection measures for web scraping.
|
||||||
|
/// </summary>
|
||||||
|
public interface IBrowserService : IDisposable
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a new browser context with proper configuration for BOM website scraping.
|
||||||
|
/// Configures viewport, user agent, timezone, and HTTP headers to mimic a real browser.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>A configured browser context ready for page creation</returns>
|
||||||
|
Task<IBrowserContext> CreateContextAsync();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a new page with anti-detection scripts and optional debug event handlers.
|
||||||
|
/// Injects comprehensive anti-detection JavaScript to avoid bot detection.
|
||||||
|
/// If debug mode is enabled, captures console messages and network requests for debugging.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="context">The browser context to create the page in</param>
|
||||||
|
/// <param name="requestId">Unique request identifier for debug file organization</param>
|
||||||
|
/// <returns>Tuple containing the page, list of console messages, and list of network requests</returns>
|
||||||
|
Task<(IPage page, List<(string type, string text, DateTime timestamp)> consoleMessages, List<(string method, string url, int? status, string resourceType, DateTime timestamp)> networkRequests)> CreatePageWithDebugAsync(
|
||||||
|
IBrowserContext context,
|
||||||
|
string requestId);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Initializes and pre-warms the browser instance.
|
||||||
|
/// Launches the Chromium browser to reduce latency on first use.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
|
||||||
|
Task InitializeAsync(CancellationToken cancellationToken = default);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the semaphore used to synchronize browser operations.
|
||||||
|
/// Ensures only one browser operation runs at a time to prevent resource conflicts.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>The semaphore for synchronizing browser access</returns>
|
||||||
|
SemaphoreSlim GetSemaphore();
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
using BomLocalService.Models;
|
||||||
|
|
||||||
|
namespace BomLocalService.Services.Interfaces;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Service interface for managing cached radar screenshot files and metadata.
|
||||||
|
/// Handles file system operations for storing and retrieving cached BOM radar screenshots.
|
||||||
|
/// </summary>
|
||||||
|
public interface ICacheService
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the cached screenshot file path and associated metadata for a location.
|
||||||
|
/// Searches for PNG files matching the location pattern and loads the corresponding metadata JSON file.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
|
||||||
|
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
|
||||||
|
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
|
||||||
|
/// <returns>Tuple containing the screenshot file path and metadata, or (null, null) if not found</returns>
|
||||||
|
Task<(string? screenshotPath, LastUpdatedInfo? metadata)> GetCachedScreenshotWithMetadataAsync(
|
||||||
|
string suburb,
|
||||||
|
string state,
|
||||||
|
CancellationToken cancellationToken = default);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Saves metadata JSON file alongside a screenshot file.
|
||||||
|
/// Creates a JSON file with the same base name as the screenshot but with .json extension.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="screenshotPath">Full path to the screenshot PNG file</param>
|
||||||
|
/// <param name="metadata">Metadata to save (observation time, forecast time, weather station, distance)</param>
|
||||||
|
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
|
||||||
|
Task SaveMetadataAsync(string screenshotPath, LastUpdatedInfo metadata, CancellationToken cancellationToken = default);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Checks if cached metadata is still valid (not expired).
|
||||||
|
/// Cache is valid if observation time + expiration buffer (typically 15.5 minutes) is still in the future.
|
||||||
|
/// BOM updates observations every 15 minutes, so the buffer accounts for timing variations.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="metadata">The metadata to validate, or null</param>
|
||||||
|
/// <returns>True if metadata exists and cache is still valid, false otherwise</returns>
|
||||||
|
bool IsCacheValid(LastUpdatedInfo? metadata);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the file system path to the most recent cached screenshot for a location.
|
||||||
|
/// Returns files ordered by creation time, most recent first.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
|
||||||
|
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
|
||||||
|
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
|
||||||
|
/// <returns>File system path to the cached screenshot, or empty string if not found</returns>
|
||||||
|
Task<string> GetCachedScreenshotPathAsync(string suburb, string state, CancellationToken cancellationToken = default);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Deletes all cached files (PNG screenshots and JSON metadata) for a location.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
|
||||||
|
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
|
||||||
|
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
|
||||||
|
/// <returns>True if any files were deleted, false if no cached files existed</returns>
|
||||||
|
Task<bool> DeleteCachedLocationAsync(string suburb, string state, CancellationToken cancellationToken = default);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the cache directory path where screenshots and metadata are stored.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>The full path to the cache directory</returns>
|
||||||
|
string GetCacheDirectory();
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
using Microsoft.Playwright;
|
||||||
|
|
||||||
|
namespace BomLocalService.Services.Interfaces;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Service interface for debug file generation during web scraping operations.
|
||||||
|
/// When enabled, saves screenshots, HTML, console logs, and network request information for troubleshooting.
|
||||||
|
/// </summary>
|
||||||
|
public interface IDebugService
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Indicates whether debug mode is enabled.
|
||||||
|
/// When false, debug methods return immediately without saving files.
|
||||||
|
/// </summary>
|
||||||
|
bool IsEnabled { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a folder for storing debug files for a specific request.
|
||||||
|
/// Folder structure: {CacheDirectory}/debug/{requestId}/
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="requestId">Unique identifier for the request (e.g., timestamp + GUID)</param>
|
||||||
|
/// <returns>Full path to the created debug folder, or empty string if debug is disabled</returns>
|
||||||
|
string CreateRequestFolder(string requestId);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Saves debug files for a scraping step.
|
||||||
|
/// Creates a step folder (e.g., "step_01_homepage_loaded") containing:
|
||||||
|
/// - screenshot.png (full page screenshot)
|
||||||
|
/// - page.html (complete HTML source)
|
||||||
|
/// - console.log (browser console messages)
|
||||||
|
/// - network.log (network request summary)
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="requestFolder">The debug folder path for this request (from CreateRequestFolder)</param>
|
||||||
|
/// <param name="stepNumber">Step number for ordering (e.g., 1, 2, 3)</param>
|
||||||
|
/// <param name="stepName">Descriptive name for the step (e.g., "homepage_loaded", "search_button_clicked")</param>
|
||||||
|
/// <param name="page">The Playwright page instance to capture</param>
|
||||||
|
/// <param name="consoleMessages">Optional list of console messages to save</param>
|
||||||
|
/// <param name="networkRequests">Optional list of network requests to save</param>
|
||||||
|
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
|
||||||
|
Task SaveStepDebugAsync(
|
||||||
|
string requestFolder,
|
||||||
|
int stepNumber,
|
||||||
|
string stepName,
|
||||||
|
IPage page,
|
||||||
|
List<(string type, string text, DateTime timestamp)>? consoleMessages = null,
|
||||||
|
List<(string method, string url, int? status, string resourceType, DateTime timestamp)>? networkRequests = null,
|
||||||
|
CancellationToken cancellationToken = default);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Saves debug files when an error occurs during scraping.
|
||||||
|
/// Creates an "error" subfolder containing:
|
||||||
|
/// - error.txt (error message and stack trace)
|
||||||
|
/// - screenshot.png (page state at error time, if page available)
|
||||||
|
/// - page.html (HTML source at error time, if page available)
|
||||||
|
/// - console.log (console messages up to error)
|
||||||
|
/// - network.log (network requests up to error)
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="requestFolder">The debug folder path for this request (from CreateRequestFolder)</param>
|
||||||
|
/// <param name="errorMessage">The error message and/or stack trace to save</param>
|
||||||
|
/// <param name="page">Optional Playwright page instance to capture error state</param>
|
||||||
|
/// <param name="consoleMessages">Optional list of console messages to save</param>
|
||||||
|
/// <param name="networkRequests">Optional list of network requests to save</param>
|
||||||
|
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
|
||||||
|
Task SaveErrorDebugAsync(
|
||||||
|
string requestFolder,
|
||||||
|
string errorMessage,
|
||||||
|
IPage? page = null,
|
||||||
|
List<(string type, string text, DateTime timestamp)>? consoleMessages = null,
|
||||||
|
List<(string method, string url, int? status, string resourceType, DateTime timestamp)>? networkRequests = null,
|
||||||
|
CancellationToken cancellationToken = default);
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
using BomLocalService.Models;
|
||||||
|
using Microsoft.Playwright;
|
||||||
|
|
||||||
|
namespace BomLocalService.Services.Interfaces;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Service interface for scraping radar screenshots from the BOM website.
|
||||||
|
/// Orchestrates the multi-step process of navigating BOM, finding locations, and capturing radar screenshots.
|
||||||
|
/// </summary>
|
||||||
|
public interface IScrapingService
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Scrapes a radar screenshot from the BOM website for a given location.
|
||||||
|
/// Performs the complete workflow: navigates to BOM, searches for location, navigates to radar page,
|
||||||
|
/// waits for map to render, captures screenshot, extracts metadata, and saves to cache.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="suburb">The suburb name (e.g., "Pomona", "Brisbane")</param>
|
||||||
|
/// <param name="state">The Australian state abbreviation (e.g., "QLD", "NSW", "VIC")</param>
|
||||||
|
/// <param name="debugFolder">Folder path for saving debug files (screenshots, HTML, logs) if debug mode is enabled</param>
|
||||||
|
/// <param name="page">The Playwright page instance to use for scraping</param>
|
||||||
|
/// <param name="consoleMessages">List to capture console messages from the browser (for debugging)</param>
|
||||||
|
/// <param name="networkRequests">List to capture network request information (for debugging)</param>
|
||||||
|
/// <param name="cancellationToken">Cancellation token to cancel the operation</param>
|
||||||
|
/// <returns>Radar screenshot response with image path and metadata</returns>
|
||||||
|
Task<RadarScreenshotResponse> ScrapeRadarScreenshotAsync(
|
||||||
|
string suburb,
|
||||||
|
string state,
|
||||||
|
string debugFolder,
|
||||||
|
IPage page,
|
||||||
|
List<(string type, string text, DateTime timestamp)> consoleMessages,
|
||||||
|
List<(string method, string url, int? status, string resourceType, DateTime timestamp)> networkRequests,
|
||||||
|
CancellationToken cancellationToken = default);
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
using BomLocalService.Models;
|
||||||
|
using Microsoft.Playwright;
|
||||||
|
|
||||||
|
namespace BomLocalService.Services.Interfaces;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Service interface for parsing time and metadata information from BOM website content.
|
||||||
|
/// Extracts observation times, forecast times, weather station names, and distances from BOM weather map pages.
|
||||||
|
/// </summary>
|
||||||
|
public interface ITimeParsingService
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Extracts last updated information from a BOM weather map page.
|
||||||
|
/// Scrapes the weather metadata section to get observation time, forecast time, weather station, and distance.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="page">The Playwright page instance containing the loaded BOM weather map</param>
|
||||||
|
/// <returns>Last updated information with parsed times and metadata</returns>
|
||||||
|
Task<LastUpdatedInfo> ExtractLastUpdatedInfoAsync(IPage page);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Parses last updated text from BOM website to extract structured information.
|
||||||
|
/// Handles various text formats like "Observations: 11 minutes ago, 8:20 pm AEST at Gympie weather station, 30 km from Pomona, QLD".
|
||||||
|
/// Converts times to UTC using the configured timezone.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="text">The raw text content from the BOM weather metadata section</param>
|
||||||
|
/// <returns>Parsed last updated information with observation time, forecast time, weather station, and distance</returns>
|
||||||
|
LastUpdatedInfo ParseLastUpdatedText(string text);
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,373 @@
|
|||||||
|
using BomLocalService.Models;
|
||||||
|
using BomLocalService.Services.Interfaces;
|
||||||
|
using BomLocalService.Utilities;
|
||||||
|
using Microsoft.Playwright;
|
||||||
|
|
||||||
|
namespace BomLocalService.Services;
|
||||||
|
|
||||||
|
public class ScrapingService : IScrapingService
|
||||||
|
{
|
||||||
|
private readonly ILogger<ScrapingService> _logger;
|
||||||
|
private readonly IBrowserService _browserService;
|
||||||
|
private readonly ITimeParsingService _timeParsingService;
|
||||||
|
private readonly ICacheService _cacheService;
|
||||||
|
private readonly IDebugService _debugService;
|
||||||
|
private readonly int _dynamicContentWaitMs;
|
||||||
|
private readonly int _tileRenderWaitMs;
|
||||||
|
|
||||||
|
// Selector constants
|
||||||
|
private static readonly string[] SearchButtonSelectors = new[]
|
||||||
|
{
|
||||||
|
"button[data-testid='searchLabel']",
|
||||||
|
"button[aria-label='Search for a location']",
|
||||||
|
"button.search-location__trigger-button",
|
||||||
|
"button.bom-button:has(span.bom-button__label:has-text('Search for a location'))",
|
||||||
|
"button:has-text('Search for a location')",
|
||||||
|
"[data-testid='showSearchModal'] button"
|
||||||
|
};
|
||||||
|
|
||||||
|
private static readonly string[] RadarLinkSelectors = new[]
|
||||||
|
{
|
||||||
|
"a.jump-link.bom-button.bom-button--secondary:has-text('Rain radar and weather map')",
|
||||||
|
".cta.button a.jump-link:has-text('Rain radar and weather map')",
|
||||||
|
"a.jump-link:has-text('Rain radar and weather map')",
|
||||||
|
".cta.button a:has-text('Rain radar and weather map')",
|
||||||
|
"a.bom-button--secondary:has-text('Rain radar and weather map')",
|
||||||
|
"a:has-text('Rain radar and weather map')",
|
||||||
|
"a:has-text('rain radar')",
|
||||||
|
"a[href*='radar']"
|
||||||
|
};
|
||||||
|
|
||||||
|
public ScrapingService(
|
||||||
|
ILogger<ScrapingService> logger,
|
||||||
|
IBrowserService browserService,
|
||||||
|
ITimeParsingService timeParsingService,
|
||||||
|
ICacheService cacheService,
|
||||||
|
IDebugService debugService,
|
||||||
|
IConfiguration configuration)
|
||||||
|
{
|
||||||
|
_logger = logger;
|
||||||
|
_browserService = browserService;
|
||||||
|
_timeParsingService = timeParsingService;
|
||||||
|
_cacheService = cacheService;
|
||||||
|
_debugService = debugService;
|
||||||
|
_dynamicContentWaitMs = configuration.GetValue<int>("Screenshot:DynamicContentWaitMs", 2000);
|
||||||
|
_tileRenderWaitMs = configuration.GetValue<int>("Screenshot:TileRenderWaitMs", 5000);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Scrapes the BOM website to get a radar screenshot for a location
|
||||||
|
/// </summary>
|
||||||
|
public async Task<RadarScreenshotResponse> ScrapeRadarScreenshotAsync(
|
||||||
|
string suburb,
|
||||||
|
string state,
|
||||||
|
string debugFolder,
|
||||||
|
IPage page,
|
||||||
|
List<(string type, string text, DateTime timestamp)> consoleMessages,
|
||||||
|
List<(string method, string url, int? status, string resourceType, DateTime timestamp)> networkRequests,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
// Step 1: Navigate to BOM homepage and wait for search button
|
||||||
|
_logger.LogInformation("Navigating to BOM homepage");
|
||||||
|
await page.GotoAsync("https://www.bom.gov.au/", new PageGotoOptions
|
||||||
|
{
|
||||||
|
WaitUntil = WaitUntilState.DOMContentLoaded,
|
||||||
|
Timeout = 30000
|
||||||
|
});
|
||||||
|
|
||||||
|
// Wait for search button to be ready instead of NetworkIdle (faster)
|
||||||
|
var searchButtonReady = page.Locator("button[data-testid='searchLabel'], button[aria-label='Search for a location'], button.search-location__trigger-button").First;
|
||||||
|
await searchButtonReady.WaitForAsync(new LocatorWaitForOptions { Timeout = 10000, State = WaitForSelectorState.Visible });
|
||||||
|
await _debugService.SaveStepDebugAsync(debugFolder, 1, "homepage_loaded", page, consoleMessages, networkRequests, cancellationToken);
|
||||||
|
|
||||||
|
// Step 2: Click "Search for a location" button to open search UI
|
||||||
|
_logger.LogInformation("Clicking 'Search for a location' button");
|
||||||
|
var searchButton = await SelectorHelper.FindLocatorBySelectorsAsync(page, SearchButtonSelectors, _logger);
|
||||||
|
|
||||||
|
if (searchButton == null)
|
||||||
|
{
|
||||||
|
var errorMsg = "Could not find 'Search for a location' button on BOM homepage.";
|
||||||
|
await _debugService.SaveErrorDebugAsync(debugFolder, errorMsg, page, consoleMessages, networkRequests, cancellationToken);
|
||||||
|
throw new Exception(errorMsg);
|
||||||
|
}
|
||||||
|
|
||||||
|
await searchButton.ClickAsync();
|
||||||
|
|
||||||
|
// Wait for search input to appear
|
||||||
|
var searchInputReady = page.Locator("#search-enter-keyword").First;
|
||||||
|
await searchInputReady.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000, State = WaitForSelectorState.Visible });
|
||||||
|
await _debugService.SaveStepDebugAsync(debugFolder, 2, "search_button_clicked", page, consoleMessages, networkRequests, cancellationToken);
|
||||||
|
|
||||||
|
// Step 3: Find and fill search box with suburb name
|
||||||
|
_logger.LogInformation("Searching for suburb: {Suburb}", suburb);
|
||||||
|
var searchInput = page.Locator("#search-enter-keyword").First;
|
||||||
|
await searchInput.FillAsync(suburb);
|
||||||
|
|
||||||
|
// Step 4: Wait for autocomplete suggestions to appear
|
||||||
|
_logger.LogInformation("Waiting for autocomplete suggestions");
|
||||||
|
await page.WaitForFunctionAsync(@"() => {
|
||||||
|
const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=""listitem""]'));
|
||||||
|
return results.length > 0 && results.some(r => r.offsetParent !== null);
|
||||||
|
}", new PageWaitForFunctionOptions { Timeout = 10000 });
|
||||||
|
await _debugService.SaveStepDebugAsync(debugFolder, 3, "search_input_filled", page, consoleMessages, networkRequests, cancellationToken);
|
||||||
|
|
||||||
|
// Step 5: Find the matching search result based on suburb and state
|
||||||
|
_logger.LogInformation("Looking for matching search result for {Suburb}, {State}", suburb, state);
|
||||||
|
var suburbLower = suburb.ToLower().Trim();
|
||||||
|
var stateLower = state.ToLower().Trim();
|
||||||
|
|
||||||
|
// Get the actual count from the summary element
|
||||||
|
var summaryText = await page.Locator("#location-results-title, [data-testid='location-results-title']").First.TextContentAsync();
|
||||||
|
int? actualCount = null;
|
||||||
|
if (!string.IsNullOrEmpty(summaryText))
|
||||||
|
{
|
||||||
|
// Parse "3 of 3 location results" or similar
|
||||||
|
var countMatch = System.Text.RegularExpressions.Regex.Match(summaryText, @"(\d+)\s+of\s+(\d+)", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
|
||||||
|
if (countMatch.Success && countMatch.Groups.Count >= 3)
|
||||||
|
{
|
||||||
|
if (int.TryParse(countMatch.Groups[2].Value, out var total))
|
||||||
|
{
|
||||||
|
actualCount = total;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fetch all result data in one JavaScript evaluation - extract location name and state separately
|
||||||
|
List<(string name, string desc, string fullText)> results = new();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var resultData = await page.EvaluateAsync<string[][]>(@"() => {
|
||||||
|
// Scope to the location results list - find the ul with aria-labelledby pointing to location-results-title
|
||||||
|
const resultsList = document.querySelector('ul[aria-labelledby=""location-results-title""]');
|
||||||
|
if (!resultsList) {
|
||||||
|
console.log('Location results list not found');
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
const results = Array.from(resultsList.querySelectorAll('li.bom-linklist__item[role=""listitem""]'));
|
||||||
|
console.log('Found', results.length, 'location results');
|
||||||
|
return results.map((r, index) => {
|
||||||
|
// Query from the li element - elements are nested inside <a> tag
|
||||||
|
const nameEl = r.querySelector('[data-testid=""location-name""]');
|
||||||
|
const descEl = r.querySelector('.bom-linklist-item__desc');
|
||||||
|
const name = nameEl ? (nameEl.textContent || nameEl.innerText || '').trim() : '';
|
||||||
|
const desc = descEl ? (descEl.textContent || descEl.innerText || '').trim() : '';
|
||||||
|
const fullText = (r.textContent || r.innerText || '').trim();
|
||||||
|
console.log('Result', index, ':', { hasNameEl: !!nameEl, hasDescEl: !!descEl, name: name, desc: desc });
|
||||||
|
return [name, desc, fullText];
|
||||||
|
});
|
||||||
|
}");
|
||||||
|
|
||||||
|
// Convert to structured data
|
||||||
|
results = resultData.Select(arr => (
|
||||||
|
name: arr.Length > 0 ? arr[0] : "",
|
||||||
|
desc: arr.Length > 1 ? arr[1] : "",
|
||||||
|
fullText: arr.Length > 2 ? arr[2] : ""
|
||||||
|
)).ToList();
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(ex, "Failed to extract structured result data, falling back to text content");
|
||||||
|
// Fallback to simple text extraction
|
||||||
|
var resultTexts = await page.EvaluateAsync<string[]>(@"() => {
|
||||||
|
const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=""listitem""]'));
|
||||||
|
return results.map(r => r.textContent || '');
|
||||||
|
}");
|
||||||
|
results = resultTexts.Select(text => (name: "", desc: "", fullText: text)).ToList();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (actualCount.HasValue)
|
||||||
|
{
|
||||||
|
_logger.LogInformation("Found {Count} location results (summary: {Summary})", actualCount.Value, summaryText?.Trim());
|
||||||
|
// Only use the first N results that match the actual count
|
||||||
|
if (results.Count > actualCount.Value)
|
||||||
|
{
|
||||||
|
results = results.Take(actualCount.Value).ToList();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_logger.LogInformation("Found {Count} search results", results.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
int? matchingIndex = null;
|
||||||
|
for (int i = 0; i < results.Count; i++)
|
||||||
|
{
|
||||||
|
var (name, desc, fullText) = results[i];
|
||||||
|
var nameLower = name.ToLower().Trim();
|
||||||
|
var descLower = desc.ToLower().Trim();
|
||||||
|
var fullTextLower = fullText.ToLower();
|
||||||
|
|
||||||
|
_logger.LogInformation("Checking result {Index}: Name='{Name}', Desc='{Desc}', FullText='{FullText}'",
|
||||||
|
i, name, desc, fullText.Length > 100 ? fullText.Substring(0, 100) + "..." : fullText);
|
||||||
|
|
||||||
|
// Check if suburb name matches (from location-name element or fallback to fullText)
|
||||||
|
var matchesSuburb = false;
|
||||||
|
if (!string.IsNullOrEmpty(name))
|
||||||
|
{
|
||||||
|
matchesSuburb = nameLower == suburbLower || nameLower.Contains(suburbLower);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// Fallback to fullText if name extraction failed
|
||||||
|
matchesSuburb = fullTextLower.Contains(suburbLower);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if state matches (from description which contains "Queensland 4300" or "New South Wales 2469")
|
||||||
|
var matchesState = false;
|
||||||
|
if (!string.IsNullOrEmpty(desc))
|
||||||
|
{
|
||||||
|
matchesState = StateAbbreviationHelper.MatchesState(descLower, stateLower);
|
||||||
|
}
|
||||||
|
// Always check fullText for state as fallback
|
||||||
|
if (!matchesState)
|
||||||
|
{
|
||||||
|
matchesState = StateAbbreviationHelper.MatchesState(fullTextLower, stateLower);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogInformation("Result {Index}: matchesSuburb={MatchesSuburb} (suburb='{Suburb}' vs name='{Name}'), matchesState={MatchesState} (state='{State}' vs desc='{Desc}')",
|
||||||
|
i, matchesSuburb, suburbLower, nameLower, matchesState, stateLower, descLower);
|
||||||
|
|
||||||
|
if (matchesSuburb && matchesState)
|
||||||
|
{
|
||||||
|
matchingIndex = i;
|
||||||
|
_logger.LogInformation("Found matching result: {Name} - {Desc}", name, desc);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Click the matching result (or first if no match)
|
||||||
|
// Scope to the location results list
|
||||||
|
var resultsList = page.Locator("ul[aria-labelledby='location-results-title']");
|
||||||
|
var allResults = resultsList.Locator("li.bom-linklist__item[role='listitem']");
|
||||||
|
var resultToClick = matchingIndex.HasValue ? allResults.Nth(matchingIndex.Value) : allResults.First;
|
||||||
|
|
||||||
|
if (!matchingIndex.HasValue)
|
||||||
|
{
|
||||||
|
_logger.LogInformation("No exact match found, using first result");
|
||||||
|
}
|
||||||
|
|
||||||
|
await resultToClick.ClickAsync();
|
||||||
|
await _debugService.SaveStepDebugAsync(debugFolder, 4, "search_result_selected", page, consoleMessages, networkRequests, cancellationToken);
|
||||||
|
|
||||||
|
// Wait for forecast page to load
|
||||||
|
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15000 });
|
||||||
|
await page.WaitForTimeoutAsync(_dynamicContentWaitMs); // Brief wait for dynamic content
|
||||||
|
await _debugService.SaveStepDebugAsync(debugFolder, 5, "forecast_page_loaded", page, consoleMessages, networkRequests, cancellationToken);
|
||||||
|
|
||||||
|
// Step 6: Find and click "Rain radar and weather map" link
|
||||||
|
_logger.LogInformation("Looking for 'Rain radar and weather map' link");
|
||||||
|
var radarLink = await SelectorHelper.FindLocatorBySelectorsAsync(page, RadarLinkSelectors, _logger);
|
||||||
|
|
||||||
|
if (radarLink == null)
|
||||||
|
{
|
||||||
|
var errorMsg = $"Could not find 'Rain radar and weather map' link for {suburb}, {state}";
|
||||||
|
await _debugService.SaveErrorDebugAsync(debugFolder, errorMsg, page, consoleMessages, networkRequests, cancellationToken);
|
||||||
|
throw new Exception(errorMsg);
|
||||||
|
}
|
||||||
|
|
||||||
|
await radarLink.ClickAsync();
|
||||||
|
await _debugService.SaveStepDebugAsync(debugFolder, 6, "radar_link_clicked", page, consoleMessages, networkRequests, cancellationToken);
|
||||||
|
|
||||||
|
// Step 7: Wait for weather map page to load and map to fully render
|
||||||
|
_logger.LogInformation("Waiting for weather map page to load");
|
||||||
|
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15000 });
|
||||||
|
|
||||||
|
// Wait for the map canvas element to appear with proper dimensions
|
||||||
|
_logger.LogInformation("Waiting for map canvas element to render");
|
||||||
|
var mapCanvas = page.Locator(".esri-view-surface canvas").First;
|
||||||
|
await mapCanvas.WaitForAsync(new LocatorWaitForOptions { Timeout = 15000 });
|
||||||
|
|
||||||
|
// Wait for canvas to have valid dimensions
|
||||||
|
await page.WaitForFunctionAsync(@"() => {
|
||||||
|
const canvas = document.querySelector('.esri-view-surface canvas');
|
||||||
|
return canvas && canvas.width > 0 && canvas.height > 0 && canvas.offsetWidth > 0 && canvas.offsetHeight > 0;
|
||||||
|
}", new PageWaitForFunctionOptions { Timeout = 15000 });
|
||||||
|
|
||||||
|
_logger.LogInformation("Map canvas is ready - waiting for map to render");
|
||||||
|
|
||||||
|
// Wait for Esri map view to be ready
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await page.WaitForFunctionAsync(@"() => {
|
||||||
|
try {
|
||||||
|
const elements = document.querySelectorAll('.esri-view');
|
||||||
|
for (let el of elements) {
|
||||||
|
if (el.__view && el.__view.ready) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch(e) {}
|
||||||
|
return false;
|
||||||
|
}", new PageWaitForFunctionOptions { Timeout = 30000 });
|
||||||
|
_logger.LogInformation("Esri map view is ready");
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
_logger.LogInformation("Esri view ready check timed out, continuing with fixed wait");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Additional wait for tiles to render
|
||||||
|
await page.WaitForTimeoutAsync(_tileRenderWaitMs);
|
||||||
|
|
||||||
|
await _debugService.SaveStepDebugAsync(debugFolder, 7, "weather_map_ready", page, consoleMessages, networkRequests, cancellationToken);
|
||||||
|
|
||||||
|
// Step 8: Extract last updated information
|
||||||
|
var lastUpdatedInfo = await _timeParsingService.ExtractLastUpdatedInfoAsync(page);
|
||||||
|
|
||||||
|
// Step 9: Find the map container and take screenshot of just the map area
|
||||||
|
_logger.LogInformation("Taking screenshot of map area");
|
||||||
|
var mapContainer = page.Locator(".esri-view-surface").First;
|
||||||
|
await mapContainer.WaitForAsync(new LocatorWaitForOptions { Timeout = 10000 });
|
||||||
|
|
||||||
|
// Ensure the map container is visible and has dimensions before taking screenshot
|
||||||
|
await page.WaitForFunctionAsync(@"() => {
|
||||||
|
const container = document.querySelector('.esri-view-surface');
|
||||||
|
return container && container.offsetWidth > 0 && container.offsetHeight > 0;
|
||||||
|
}", new PageWaitForFunctionOptions { Timeout = 10000 });
|
||||||
|
|
||||||
|
// Get bounding box of map container
|
||||||
|
var boundingBox = await mapContainer.BoundingBoxAsync();
|
||||||
|
if (boundingBox == null)
|
||||||
|
{
|
||||||
|
throw new Exception("Could not determine map container bounds");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate cache filename based on suburb, state and timestamp
|
||||||
|
var locationKey = LocationHelper.GetLocationKey(suburb, state);
|
||||||
|
var safeLocationKey = LocationHelper.SanitizeFileName(locationKey);
|
||||||
|
var timestamp = DateTime.UtcNow.ToString("yyyyMMdd_HHmmss");
|
||||||
|
var cacheFileName = $"{safeLocationKey}_{timestamp}.png";
|
||||||
|
var cacheFilePath = Path.Combine(_cacheService.GetCacheDirectory(), cacheFileName);
|
||||||
|
|
||||||
|
// Take screenshot of the map area only
|
||||||
|
await page.ScreenshotAsync(new PageScreenshotOptions
|
||||||
|
{
|
||||||
|
Path = cacheFilePath,
|
||||||
|
Clip = new Clip
|
||||||
|
{
|
||||||
|
X = boundingBox.X,
|
||||||
|
Y = boundingBox.Y,
|
||||||
|
Width = boundingBox.Width,
|
||||||
|
Height = boundingBox.Height
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
_logger.LogInformation("Screenshot saved to: {Path}", cacheFilePath);
|
||||||
|
|
||||||
|
// Save metadata alongside the screenshot
|
||||||
|
await _cacheService.SaveMetadataAsync(cacheFilePath, lastUpdatedInfo, cancellationToken);
|
||||||
|
|
||||||
|
return ResponseBuilder.CreateRadarScreenshotResponse(cacheFilePath, lastUpdatedInfo);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
// Save error debug info if debug is enabled
|
||||||
|
await _debugService.SaveErrorDebugAsync(debugFolder, $"Exception: {ex.Message}\n\nStackTrace:\n{ex.StackTrace}", page, consoleMessages, networkRequests, cancellationToken);
|
||||||
|
_logger.LogError(ex, "Error during radar screenshot capture");
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,254 @@
|
|||||||
|
using BomLocalService.Models;
|
||||||
|
using BomLocalService.Services.Interfaces;
|
||||||
|
using Microsoft.Playwright;
|
||||||
|
using System.Globalization;
|
||||||
|
using System.Text.RegularExpressions;
|
||||||
|
|
||||||
|
namespace BomLocalService.Services;
|
||||||
|
|
||||||
|
public class TimeParsingService : ITimeParsingService
|
||||||
|
{
|
||||||
|
private readonly ILogger<TimeParsingService> _logger;
|
||||||
|
private readonly TimeZoneInfo _timeZoneInfo;
|
||||||
|
|
||||||
|
public TimeParsingService(ILogger<TimeParsingService> logger, IConfiguration configuration)
|
||||||
|
{
|
||||||
|
_logger = logger;
|
||||||
|
var timezone = configuration.GetValue<string>("Timezone") ?? "Australia/Brisbane";
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById(timezone);
|
||||||
|
_logger.LogInformation("Using timezone: {Timezone} ({DisplayName})", timezone, _timeZoneInfo.DisplayName);
|
||||||
|
}
|
||||||
|
catch (TimeZoneNotFoundException)
|
||||||
|
{
|
||||||
|
_logger.LogWarning("Timezone {Timezone} not found, defaulting to Australia/Brisbane", timezone);
|
||||||
|
_timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById("Australia/Brisbane");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Extracts last updated information from the weather map page
|
||||||
|
/// </summary>
|
||||||
|
public async Task<LastUpdatedInfo> ExtractLastUpdatedInfoAsync(IPage page)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
// Look for the weather metadata section using the correct selector
|
||||||
|
// HTML structure: <section data-testid="weatherMetadata"> with aria-label="Last updated"
|
||||||
|
var metadataSection = page.Locator("section[data-testid='weatherMetadata'], section[aria-label='Last updated']").First;
|
||||||
|
|
||||||
|
// Extract text from each div separately to preserve structure
|
||||||
|
// Structure: <div>Observations: X minutes ago, 9:40 pm AEST</div><div>at Gympie...</div><div>Forecast: ...</div>
|
||||||
|
var lastUpdatedText = await page.EvaluateAsync<string>(@"() => {
|
||||||
|
const section = document.querySelector('section[data-testid=""weatherMetadata""]') ||
|
||||||
|
document.querySelector('section[aria-label=""Last updated""]');
|
||||||
|
if (!section) return null;
|
||||||
|
|
||||||
|
// Get all divs and join with newlines to preserve structure
|
||||||
|
const divs = section.querySelectorAll('div');
|
||||||
|
return Array.from(divs).map(div => div.textContent.trim()).filter(text => text).join(' ');
|
||||||
|
}");
|
||||||
|
|
||||||
|
if (string.IsNullOrEmpty(lastUpdatedText))
|
||||||
|
{
|
||||||
|
// Fallback: get all text content
|
||||||
|
lastUpdatedText = await metadataSection.TextContentAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
return ParseLastUpdatedText(lastUpdatedText ?? string.Empty);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(ex, "Failed to extract last updated information");
|
||||||
|
return new LastUpdatedInfo
|
||||||
|
{
|
||||||
|
ObservationTime = DateTime.UtcNow,
|
||||||
|
ForecastTime = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Parses last updated text to extract observation time, forecast time, weather station, and distance
|
||||||
|
/// </summary>
|
||||||
|
public LastUpdatedInfo ParseLastUpdatedText(string text)
|
||||||
|
{
|
||||||
|
var info = new LastUpdatedInfo();
|
||||||
|
_logger.LogInformation("Parsing last updated text: {Text}", text);
|
||||||
|
|
||||||
|
// Try to parse "Observations: 11 minutes ago, 8:20 pm AEST at Gympie weather station, 30 km from Pomona, QLD"
|
||||||
|
// And "Forecast: 41 minutes ago, 7:50 pm AEST"
|
||||||
|
// Prefer parsing the actual time string over "minutes ago" for accuracy
|
||||||
|
|
||||||
|
// Parse observation time - improved regex to better capture time and timezone
|
||||||
|
// Stop timezone capture before "at" (e.g., "AEST at Gympie" or "AESTat Gympie" should capture just "AEST")
|
||||||
|
// Pattern: "Observations: X minutes ago, 9:40 pm AESTat Gympie..." (note: sometimes no space between AEST and "at")
|
||||||
|
// Capture timezone as 3-4 uppercase letters (AEST, AEDT, etc.) optionally followed by "at"
|
||||||
|
var observationMatch = Regex.Match(text, @"Observations:\s*(?:\d+\s*minutes?\s*ago)?[,\s]+([\d:]+(?:\s*[ap]m)?)\s+([A-Z]{3,4})(?:at|\s+at|,|$)", RegexOptions.IgnoreCase);
|
||||||
|
_logger.LogDebug("Observation regex match result: Success={Success}, Groups.Count={Count}", observationMatch.Success, observationMatch.Groups.Count);
|
||||||
|
if (observationMatch.Success && observationMatch.Groups.Count >= 3 && observationMatch.Groups[1].Success)
|
||||||
|
{
|
||||||
|
var timeStr = observationMatch.Groups[1].Value.Trim();
|
||||||
|
var timezoneStr = observationMatch.Groups[2].Success ? observationMatch.Groups[2].Value.Trim() : null;
|
||||||
|
|
||||||
|
// Clean up timezone string - remove "at" if it got concatenated (e.g., "AESTat" -> "AEST")
|
||||||
|
if (!string.IsNullOrEmpty(timezoneStr) && timezoneStr.EndsWith("at", StringComparison.OrdinalIgnoreCase) && timezoneStr.Length > 2)
|
||||||
|
{
|
||||||
|
timezoneStr = timezoneStr.Substring(0, timezoneStr.Length - 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("Extracted observation time string: '{TimeStr}', timezone: '{TzStr}'", timeStr, timezoneStr ?? "null");
|
||||||
|
|
||||||
|
if (TryParseTimeString(timeStr, timezoneStr, out var observationTime))
|
||||||
|
{
|
||||||
|
info.ObservationTime = observationTime;
|
||||||
|
_logger.LogInformation("Successfully parsed observation time: {Time} UTC (from '{TimeStr}' {TzStr})",
|
||||||
|
observationTime, timeStr, timezoneStr ?? "default timezone");
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// Fallback to "minutes ago" if time parsing fails
|
||||||
|
_logger.LogWarning("Failed to parse observation time from '{TimeStr}', falling back to 'minutes ago'", timeStr);
|
||||||
|
var minutesAgoMatch = Regex.Match(text, @"Observations:\s*(\d+)\s*minutes?\s*ago", RegexOptions.IgnoreCase);
|
||||||
|
if (minutesAgoMatch.Success && int.TryParse(minutesAgoMatch.Groups[1].Value, out var minutesAgo))
|
||||||
|
{
|
||||||
|
info.ObservationTime = DateTime.UtcNow.AddMinutes(-minutesAgo);
|
||||||
|
_logger.LogInformation("Used 'minutes ago' fallback: {MinutesAgo} minutes ago = {Time} UTC",
|
||||||
|
minutesAgo, info.ObservationTime);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse forecast time
|
||||||
|
var forecastMatch = Regex.Match(text, @"Forecast:\s*(?:\d+\s*minutes?\s*ago)?[,\s]*([\d:]+(?:\s*[ap]m)?)\s+([A-Z]+)(?:\s+at|$)", RegexOptions.IgnoreCase);
|
||||||
|
if (forecastMatch.Success && forecastMatch.Groups[1].Success)
|
||||||
|
{
|
||||||
|
var timeStr = forecastMatch.Groups[1].Value.Trim();
|
||||||
|
var timezoneStr = forecastMatch.Groups[2].Success ? forecastMatch.Groups[2].Value.Trim() : null;
|
||||||
|
|
||||||
|
if (TryParseTimeString(timeStr, timezoneStr, out var forecastTime))
|
||||||
|
{
|
||||||
|
info.ForecastTime = forecastTime;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// Fallback to "minutes ago" if time parsing fails
|
||||||
|
var minutesAgoMatch = Regex.Match(text, @"Forecast:\s*(\d+)\s*minutes?\s*ago", RegexOptions.IgnoreCase);
|
||||||
|
if (minutesAgoMatch.Success && int.TryParse(minutesAgoMatch.Groups[1].Value, out var minutesAgo))
|
||||||
|
{
|
||||||
|
info.ForecastTime = DateTime.UtcNow.AddMinutes(-minutesAgo);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract weather station name
|
||||||
|
var stationMatch = Regex.Match(text, @"at\s+([^,]+)\s+weather\s+station", RegexOptions.IgnoreCase);
|
||||||
|
if (stationMatch.Success)
|
||||||
|
{
|
||||||
|
info.WeatherStation = stationMatch.Groups[1].Value.Trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract distance
|
||||||
|
var distanceMatch = Regex.Match(text, @"(\d+)\s*km\s+from", RegexOptions.IgnoreCase);
|
||||||
|
if (distanceMatch.Success)
|
||||||
|
{
|
||||||
|
info.Distance = $"{distanceMatch.Groups[1].Value} km";
|
||||||
|
}
|
||||||
|
|
||||||
|
return info;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Tries to parse a time string with optional timezone to UTC
|
||||||
|
/// </summary>
|
||||||
|
private bool TryParseTimeString(string timeStr, string? timezoneStr, out DateTime utcTime)
|
||||||
|
{
|
||||||
|
utcTime = DateTime.UtcNow;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
// Parse time formats like "8:20 pm", "8:20pm", "20:20"
|
||||||
|
var timeFormats = new[] { "h:mm tt", "h:mmtt", "HH:mm", "H:mm tt", "H:mmtt", "H:mm" };
|
||||||
|
DateTime localTime = default;
|
||||||
|
|
||||||
|
// Try parsing with various formats
|
||||||
|
bool parsed = false;
|
||||||
|
foreach (var format in timeFormats)
|
||||||
|
{
|
||||||
|
if (DateTime.TryParseExact(timeStr, format, CultureInfo.InvariantCulture, DateTimeStyles.None, out var parsedTime))
|
||||||
|
{
|
||||||
|
localTime = parsedTime;
|
||||||
|
parsed = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!parsed)
|
||||||
|
{
|
||||||
|
// Fallback to general parsing
|
||||||
|
if (!DateTime.TryParse(timeStr, CultureInfo.InvariantCulture, DateTimeStyles.None, out localTime))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Determine the timezone - use configured timezone if not specified or if it's AEST/AEDT
|
||||||
|
TimeZoneInfo tz = _timeZoneInfo;
|
||||||
|
if (!string.IsNullOrEmpty(timezoneStr))
|
||||||
|
{
|
||||||
|
// AEST/AEDT are typically the same as Australia/Brisbane or Australia/Sydney
|
||||||
|
// Both Brisbane and Sydney use AEST/AEDT
|
||||||
|
if (timezoneStr.Contains("AEST", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
timezoneStr.Contains("AEDT", StringComparison.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
tz = _timeZoneInfo; // Use configured timezone
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// Try to find the timezone, fallback to configured
|
||||||
|
try
|
||||||
|
{
|
||||||
|
tz = TimeZoneInfo.FindSystemTimeZoneById(timezoneStr);
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
tz = _timeZoneInfo;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get current time in the target timezone to determine the date
|
||||||
|
var nowInTz = TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, tz);
|
||||||
|
var dateTimeInTz = nowInTz.Date.Add(localTime.TimeOfDay);
|
||||||
|
|
||||||
|
// If the parsed time is in the future (e.g., it's 9pm but we parsed 8pm and it's actually yesterday),
|
||||||
|
// or if it's more than 12 hours in the past, assume it's from yesterday
|
||||||
|
if (dateTimeInTz > nowInTz)
|
||||||
|
{
|
||||||
|
// Time is in the future, so it must be from yesterday
|
||||||
|
dateTimeInTz = dateTimeInTz.AddDays(-1);
|
||||||
|
}
|
||||||
|
else if ((nowInTz - dateTimeInTz).TotalHours > 12)
|
||||||
|
{
|
||||||
|
// More than 12 hours ago, likely from yesterday
|
||||||
|
dateTimeInTz = dateTimeInTz.AddDays(-1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert to UTC
|
||||||
|
// Note: Brisbane (Australia/Brisbane) is always UTC+10 (AEST), no daylight saving
|
||||||
|
utcTime = TimeZoneInfo.ConvertTimeToUtc(dateTimeInTz, tz);
|
||||||
|
|
||||||
|
_logger.LogInformation("Parsed time string '{TimeStr}' with timezone '{TzStr}' as {DateTimeInTz} local ({UtcTime} UTC)",
|
||||||
|
timeStr, timezoneStr ?? "default", dateTimeInTz, utcTime);
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
namespace BomLocalService.Utilities;
|
||||||
|
|
||||||
|
public static class FilePathHelper
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the cache directory path from configuration or returns default
|
||||||
|
/// </summary>
|
||||||
|
public static string GetCacheDirectory(IConfiguration configuration)
|
||||||
|
{
|
||||||
|
return configuration.GetValue<string>("CacheDirectory")
|
||||||
|
?? Path.Combine(AppContext.BaseDirectory, "cache");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the metadata file path for a given screenshot path
|
||||||
|
/// </summary>
|
||||||
|
public static string GetMetadataFilePath(string screenshotPath)
|
||||||
|
{
|
||||||
|
return Path.ChangeExtension(screenshotPath, ".json");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the cache file pattern for a location key (e.g., "Pomona_QLD_*.png")
|
||||||
|
/// </summary>
|
||||||
|
public static string GetCacheFilePattern(string locationKey)
|
||||||
|
{
|
||||||
|
var safeLocationKey = LocationHelper.SanitizeFileName(locationKey);
|
||||||
|
return $"{safeLocationKey}_*.png";
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the cache file pattern for suburb and state
|
||||||
|
/// </summary>
|
||||||
|
public static string GetCacheFilePattern(string suburb, string state)
|
||||||
|
{
|
||||||
|
var locationKey = LocationHelper.GetLocationKey(suburb, state);
|
||||||
|
return GetCacheFilePattern(locationKey);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
namespace BomLocalService.Utilities;
|
||||||
|
|
||||||
|
public static class LocationHelper
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a location key from suburb and state (e.g., "Pomona_QLD")
|
||||||
|
/// </summary>
|
||||||
|
public static string GetLocationKey(string suburb, string state)
|
||||||
|
{
|
||||||
|
return $"{suburb}_{state}";
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Sanitizes a filename by removing invalid characters and replacing spaces/commas
|
||||||
|
/// </summary>
|
||||||
|
public static string SanitizeFileName(string fileName)
|
||||||
|
{
|
||||||
|
var invalidChars = Path.GetInvalidFileNameChars();
|
||||||
|
return string.Join("_", fileName.Split(invalidChars, StringSplitOptions.RemoveEmptyEntries))
|
||||||
|
.Replace(" ", "_")
|
||||||
|
.Replace(",", "");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Parses suburb and state from a cache filename (format: Suburb_State_YYYYMMDD_HHMMSS.png)
|
||||||
|
/// Returns null if parsing fails
|
||||||
|
/// </summary>
|
||||||
|
public static (string suburb, string state)? ParseLocationFromFilename(string fileName)
|
||||||
|
{
|
||||||
|
var fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileName);
|
||||||
|
var parts = fileNameWithoutExtension.Split('_');
|
||||||
|
|
||||||
|
if (parts.Length >= 2)
|
||||||
|
{
|
||||||
|
return (parts[0], parts[1]);
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Parses timestamp from a cache filename (format: Suburb_State_YYYYMMDD_HHMMSS.png)
|
||||||
|
/// Returns null if parsing fails
|
||||||
|
/// </summary>
|
||||||
|
public static DateTime? ParseTimestampFromFilename(string fileName)
|
||||||
|
{
|
||||||
|
var fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileName);
|
||||||
|
var parts = fileNameWithoutExtension.Split('_');
|
||||||
|
|
||||||
|
if (parts.Length >= 3)
|
||||||
|
{
|
||||||
|
// Last two parts should be date and time
|
||||||
|
var dateTimeStr = $"{parts[^2]}_{parts[^1]}";
|
||||||
|
if (DateTime.TryParseExact(dateTimeStr, "yyyyMMdd_HHmmss", null, System.Globalization.DateTimeStyles.None, out var fileTime))
|
||||||
|
{
|
||||||
|
return fileTime;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
using BomLocalService.Models;
|
||||||
|
|
||||||
|
namespace BomLocalService.Utilities;
|
||||||
|
|
||||||
|
public static class ResponseBuilder
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a RadarScreenshotResponse from a file path and metadata
|
||||||
|
/// </summary>
|
||||||
|
public static RadarScreenshotResponse CreateRadarScreenshotResponse(
|
||||||
|
string imagePath,
|
||||||
|
LastUpdatedInfo? metadata = null)
|
||||||
|
{
|
||||||
|
var lastWriteTime = File.Exists(imagePath)
|
||||||
|
? File.GetLastWriteTime(imagePath)
|
||||||
|
: DateTime.UtcNow;
|
||||||
|
|
||||||
|
if (metadata == null)
|
||||||
|
{
|
||||||
|
return new RadarScreenshotResponse
|
||||||
|
{
|
||||||
|
ImagePath = imagePath,
|
||||||
|
LastUpdated = lastWriteTime,
|
||||||
|
ObservationTime = DateTime.UtcNow,
|
||||||
|
ForecastTime = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return new RadarScreenshotResponse
|
||||||
|
{
|
||||||
|
ImagePath = imagePath,
|
||||||
|
LastUpdated = lastWriteTime,
|
||||||
|
ObservationTime = metadata.ObservationTime,
|
||||||
|
ForecastTime = metadata.ForecastTime,
|
||||||
|
WeatherStation = metadata.WeatherStation,
|
||||||
|
Distance = metadata.Distance
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
using Microsoft.Playwright;
|
||||||
|
|
||||||
|
namespace BomLocalService.Utilities;
|
||||||
|
|
||||||
|
public static class SelectorHelper
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Tries to find a locator by trying multiple selectors in order
|
||||||
|
/// Returns the first visible locator found, or null if none found
|
||||||
|
/// </summary>
|
||||||
|
public static async Task<ILocator?> FindLocatorBySelectorsAsync(
|
||||||
|
IPage page,
|
||||||
|
string[] selectors,
|
||||||
|
ILogger? logger = null)
|
||||||
|
{
|
||||||
|
foreach (var selector in selectors)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var locator = page.Locator(selector).First;
|
||||||
|
if (await locator.IsVisibleAsync())
|
||||||
|
{
|
||||||
|
logger?.LogInformation("Found element with selector: {Selector}", selector);
|
||||||
|
return locator;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// Continue to next selector
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
namespace BomLocalService.Utilities;
|
||||||
|
|
||||||
|
public static class StateAbbreviationHelper
|
||||||
|
{
|
||||||
|
private static readonly Dictionary<string, string[]> StateMappings = new(StringComparer.OrdinalIgnoreCase)
|
||||||
|
{
|
||||||
|
{ "qld", new[] { "queensland" } },
|
||||||
|
{ "nsw", new[] { "new south wales" } },
|
||||||
|
{ "vic", new[] { "victoria" } },
|
||||||
|
{ "sa", new[] { "south australia" } },
|
||||||
|
{ "wa", new[] { "western australia" } },
|
||||||
|
{ "tas", new[] { "tasmania" } },
|
||||||
|
{ "nt", new[] { "northern territory" } },
|
||||||
|
{ "act", new[] { "australian capital territory" } }
|
||||||
|
};
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Checks if the text matches the state abbreviation or full state name
|
||||||
|
/// </summary>
|
||||||
|
public static bool MatchesState(string text, string stateAbbreviation)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(stateAbbreviation))
|
||||||
|
{
|
||||||
|
return true; // No state specified, match any
|
||||||
|
}
|
||||||
|
|
||||||
|
var textLower = text.ToLower();
|
||||||
|
var stateLower = stateAbbreviation.ToLower().Trim();
|
||||||
|
|
||||||
|
// Direct match
|
||||||
|
if (textLower.Contains(stateLower))
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check abbreviation mappings
|
||||||
|
if (StateMappings.TryGetValue(stateLower, out var fullNames))
|
||||||
|
{
|
||||||
|
return fullNames.Any(fullName => textLower.Contains(fullName));
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
namespace BomLocalService.Utilities;
|
||||||
|
|
||||||
|
public static class ValidationHelper
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Validates suburb and state parameters
|
||||||
|
/// Returns error message if validation fails, null if valid
|
||||||
|
/// </summary>
|
||||||
|
public static string? ValidateLocation(string suburb, string state)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(suburb))
|
||||||
|
{
|
||||||
|
return "Suburb is required";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (string.IsNullOrWhiteSpace(state))
|
||||||
|
{
|
||||||
|
return "State is required";
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
{
|
||||||
|
"Logging": {
|
||||||
|
"LogLevel": {
|
||||||
|
"Default": "Information",
|
||||||
|
"Microsoft.AspNetCore": "Warning"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"AllowedHosts": "*",
|
||||||
|
"CacheDirectory": "./cache",
|
||||||
|
"CacheRetentionHours": 24,
|
||||||
|
"CacheExpirationMinutes": 12.5,
|
||||||
|
"CacheManagement": {
|
||||||
|
"CheckIntervalMinutes": 5,
|
||||||
|
"InitialDelaySeconds": 10,
|
||||||
|
"UpdateStaggerSeconds": 2,
|
||||||
|
"LocationStaggerSeconds": 1
|
||||||
|
},
|
||||||
|
"CacheCleanup": {
|
||||||
|
"IntervalHours": 1
|
||||||
|
},
|
||||||
|
"Screenshot": {
|
||||||
|
"DynamicContentWaitMs": 2000,
|
||||||
|
"TileRenderWaitMs": 5000
|
||||||
|
},
|
||||||
|
"Debug": {
|
||||||
|
"Enabled": false,
|
||||||
|
"WaitMs": 2000
|
||||||
|
},
|
||||||
|
"Timezone": "Australia/Brisbane"
|
||||||
|
}
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
services:
|
||||||
|
bom-local-service:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
container_name: bom-local-service
|
||||||
|
ports:
|
||||||
|
- "${HOST_PORT:-8082}:${CONTAINER_PORT:-8080}"
|
||||||
|
volumes:
|
||||||
|
- ./cache:/app/cache
|
||||||
|
environment:
|
||||||
|
# ASP.NET Core configuration
|
||||||
|
# Default to Production for security (use Development only for local debugging)
|
||||||
|
- ASPNETCORE_ENVIRONMENT=${ASPNETCORE_ENVIRONMENT:-Production}
|
||||||
|
- ASPNETCORE_URLS=http://+:${CONTAINER_PORT:-8080}
|
||||||
|
|
||||||
|
# Virtual display for non-headless browser
|
||||||
|
- DISPLAY=:99
|
||||||
|
|
||||||
|
# Application configuration (override appsettings.json)
|
||||||
|
# Use double underscore (__) for nested config keys
|
||||||
|
- CACHEDIRECTORY=${CACHEDIRECTORY:-/app/cache}
|
||||||
|
- CACHERETENTIONHOURS=${CACHERETENTIONHOURS:-24}
|
||||||
|
- CACHEEXPIRATIONMINUTES=${CACHEEXPIRATIONMINUTES:-12.5}
|
||||||
|
- CACHEMANAGEMENT__CHECKINTERVALMINUTES=${CACHEMANAGEMENT__CHECKINTERVALMINUTES:-5}
|
||||||
|
- CACHEMANAGEMENT__INITIALDELAYSECONDS=${CACHEMANAGEMENT__INITIALDELAYSECONDS:-10}
|
||||||
|
- CACHEMANAGEMENT__UPDATESTAGGERSECONDS=${CACHEMANAGEMENT__UPDATESTAGGERSECONDS:-2}
|
||||||
|
- CACHEMANAGEMENT__LOCATIONSTAGGERSECONDS=${CACHEMANAGEMENT__LOCATIONSTAGGERSECONDS:-1}
|
||||||
|
- CACHECLEANUP__INTERVALHOURS=${CACHECLEANUP__INTERVALHOURS:-1}
|
||||||
|
- SCREENSHOT__DYNAMICCONTENTWAITMS=${SCREENSHOT__DYNAMICCONTENTWAITMS:-2000}
|
||||||
|
- SCREENSHOT__TILERENDERWAITMS=${SCREENSHOT__TILERENDERWAITMS:-5000}
|
||||||
|
- DEBUG__ENABLED=${DEBUG__ENABLED:-false}
|
||||||
|
- DEBUG__WAITMS=${DEBUG__WAITMS:-2000}
|
||||||
|
- TIMEZONE=${TIMEZONE:-Australia/Brisbane}
|
||||||
|
shm_size: '1gb'
|
||||||
|
ipc: host
|
||||||
|
restart: unless-stopped
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
reservations:
|
||||||
|
devices:
|
||||||
|
- driver: nvidia
|
||||||
|
count: 1
|
||||||
|
capabilities: [gpu]
|
||||||
|
|
||||||
Reference in New Issue
Block a user