From 12dd0eedadc2e90509c35b55bdb4f7fbf9e13ca2 Mon Sep 17 00:00:00 2001 From: Alex Hope-O'Connor Date: Thu, 18 Dec 2025 00:54:57 +1000 Subject: [PATCH] Refactor scraping system to workflow-based architecture Major architectural changes: - Replace monolithic ScrapingService with workflow-based system - Extract scraping logic into discrete, testable steps - Add configuration-driven selectors, JavaScript templates, and text patterns - Implement step registry and workflow factory for extensibility - Add PageState tracking and step dependencies - Make IWorkflow generic to support different response types New components: - Scraping steps organized by category (Navigation, Search, Map, Metadata, Capture) - BaseScrapingStep base class with common dependencies - ScrapingContext for shared state between steps - RadarScrapingWorkflow orchestrating step execution - SelectorService for configurable element selection - Configuration models (SelectorConfig, TextPatternsConfig, JavaScriptTemplatesConfig) Performance and monitoring: - Add step-level timing metrics with historical tracking - Log step durations and compare to averages - Warn on slow steps/workflows (50%+ slower than average) - Integrate metrics with CacheService Configuration improvements: - Externalize all selectors to appsettings.json - Externalize JavaScript templates to appsettings.json - Externalize text patterns to appsettings.json - Support Docker configuration via mounted appsettings.json - Remove hardcoded waits, use configurable timeouts Bug fixes: - Fix DebugService collection modification exception - Fix ResetToFirstFrameStep click interception with JavaScript fallback - Remove minutes ago fallback, use timestamp parsing only - Update timestamp parsing for new BOM website format Documentation: - Update README with new architecture details - Add configuration section with Docker guidance - Add development guidelines for extending scraping system - Add performance monitoring documentation --- Dockerfile | 5 +- Models/JavaScriptTemplatesConfig.cs | 186 +++ Models/ScrapingSelectorsConfig.cs | 26 + Models/ScrapingStepConfig.cs | 18 + Models/ScrapingWorkflowConfig.cs | 11 + Models/SelectorConfig.cs | 33 + Models/TextPatternsConfig.cs | 23 + Program.cs | 56 + README.md | 280 ++++- Services/CacheService.cs | 58 +- Services/DebugService.cs | 16 +- Services/Interfaces/ICacheService.cs | 27 + Services/Interfaces/ISelectorService.cs | 21 + Services/Scraping/IScrapingStep.cs | 28 + Services/Scraping/IScrapingStepRegistry.cs | 12 + Services/Scraping/IWorkflow.cs | 14 + Services/Scraping/IWorkflowFactory.cs | 10 + Services/Scraping/PageState.cs | 18 + Services/Scraping/ScrapingContext.cs | 42 + Services/Scraping/ScrapingStepRegistry.cs | 33 + Services/Scraping/ScrapingStepResult.cs | 15 + Services/Scraping/Steps/BaseScrapingStep.cs | 64 ++ .../Steps/Capture/CaptureFramesStep.cs | 489 ++++++++ .../Steps/Map/CalculateMapBoundsStep.cs | 69 ++ Services/Scraping/Steps/Map/PauseRadarStep.cs | 66 ++ .../Steps/Map/ResetToFirstFrameStep.cs | 103 ++ .../Scraping/Steps/Map/WaitForMapReadyStep.cs | 73 ++ .../Steps/Metadata/ExtractMetadataStep.cs | 68 ++ .../Steps/Navigation/ClickRadarLinkStep.cs | 56 + .../Steps/Navigation/ClickSearchButtonStep.cs | 65 ++ .../Steps/Navigation/NavigateHomepageStep.cs | 62 + .../Steps/Search/FillSearchInputStep.cs | 48 + .../Steps/Search/SelectSearchResultStep.cs | 192 ++++ .../Steps/Search/WaitForSearchResultsStep.cs | 47 + Services/Scraping/WorkflowFactory.cs | 22 + .../Workflows/RadarScrapingWorkflow.cs | 210 ++++ .../Workflows/TemperatureMapWorkflow.cs | 41 + Services/ScrapingService.cs | 1022 +---------------- Services/SelectorService.cs | 60 + appsettings.json | 199 +++- 40 files changed, 2868 insertions(+), 1020 deletions(-) create mode 100644 Models/JavaScriptTemplatesConfig.cs create mode 100644 Models/ScrapingSelectorsConfig.cs create mode 100644 Models/ScrapingStepConfig.cs create mode 100644 Models/ScrapingWorkflowConfig.cs create mode 100644 Models/SelectorConfig.cs create mode 100644 Models/TextPatternsConfig.cs create mode 100644 Services/Interfaces/ISelectorService.cs create mode 100644 Services/Scraping/IScrapingStep.cs create mode 100644 Services/Scraping/IScrapingStepRegistry.cs create mode 100644 Services/Scraping/IWorkflow.cs create mode 100644 Services/Scraping/IWorkflowFactory.cs create mode 100644 Services/Scraping/PageState.cs create mode 100644 Services/Scraping/ScrapingContext.cs create mode 100644 Services/Scraping/ScrapingStepRegistry.cs create mode 100644 Services/Scraping/ScrapingStepResult.cs create mode 100644 Services/Scraping/Steps/BaseScrapingStep.cs create mode 100644 Services/Scraping/Steps/Capture/CaptureFramesStep.cs create mode 100644 Services/Scraping/Steps/Map/CalculateMapBoundsStep.cs create mode 100644 Services/Scraping/Steps/Map/PauseRadarStep.cs create mode 100644 Services/Scraping/Steps/Map/ResetToFirstFrameStep.cs create mode 100644 Services/Scraping/Steps/Map/WaitForMapReadyStep.cs create mode 100644 Services/Scraping/Steps/Metadata/ExtractMetadataStep.cs create mode 100644 Services/Scraping/Steps/Navigation/ClickRadarLinkStep.cs create mode 100644 Services/Scraping/Steps/Navigation/ClickSearchButtonStep.cs create mode 100644 Services/Scraping/Steps/Navigation/NavigateHomepageStep.cs create mode 100644 Services/Scraping/Steps/Search/FillSearchInputStep.cs create mode 100644 Services/Scraping/Steps/Search/SelectSearchResultStep.cs create mode 100644 Services/Scraping/Steps/Search/WaitForSearchResultsStep.cs create mode 100644 Services/Scraping/WorkflowFactory.cs create mode 100644 Services/Scraping/Workflows/RadarScrapingWorkflow.cs create mode 100644 Services/Scraping/Workflows/TemperatureMapWorkflow.cs create mode 100644 Services/SelectorService.cs diff --git a/Dockerfile b/Dockerfile index e32d2c3..a228668 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ RUN apt-get update && \ chmod +x dotnet-install.sh && \ ./dotnet-install.sh --channel 9.0 --install-dir /usr/share/dotnet && \ rm dotnet-install.sh && \ - apt-get install -y xvfb x11vnc fluxbox && \ + apt-get install -y xvfb x11vnc fluxbox curl && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -45,4 +45,7 @@ LABEL org.opencontainers.image.licenses=MIT EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \ + CMD curl --fail http://localhost:8080/api/health || exit 1 + ENTRYPOINT ["/app/start.sh"] diff --git a/Models/JavaScriptTemplatesConfig.cs b/Models/JavaScriptTemplatesConfig.cs new file mode 100644 index 0000000..883e436 --- /dev/null +++ b/Models/JavaScriptTemplatesConfig.cs @@ -0,0 +1,186 @@ +namespace BomLocalService.Models; + +/// +/// JavaScript code templates used for page evaluation +/// +public class JavaScriptTemplatesConfig +{ + public string WaitForSearchResults { get; set; } = @"() => { + const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=""listitem""]')); + return results.length > 0 && results.some(r => r.offsetParent !== null); + }"; + + public string ExtractSearchResults { get; set; } = @"() => { + const resultsList = document.querySelector('ul[aria-labelledby=""location-results-title""]'); + if (!resultsList) { + console.log('Location results list not found'); + return []; + } + const results = Array.from(resultsList.querySelectorAll('li.bom-linklist__item[role=""listitem""]')); + console.log('Found', results.length, 'location results'); + return results.map((r) => { + const nameEl = r.querySelector('[data-testid=""location-name""]'); + const descEl = r.querySelector('.bom-linklist-item__desc'); + const name = nameEl ? (nameEl.textContent || nameEl.innerText || '').trim() : ''; + const desc = descEl ? (descEl.textContent || descEl.innerText || '').trim() : ''; + const fullText = (r.textContent || r.innerText || '').trim(); + console.log('Result:', { hasNameEl: !!nameEl, hasDescEl: !!descEl, name: name, desc: desc }); + return [name, desc, fullText]; + }); + }"; + + public string ExtractSearchResultsFallback { get; set; } = @"() => { + const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=""listitem""]')); + return results.map(r => r.textContent || ''); + }"; + + public string WaitForMapCanvas { get; set; } = @"() => { + const canvas = document.querySelector('.esri-view-surface canvas'); + return canvas && canvas.width > 0 && canvas.height > 0 && canvas.offsetWidth > 0 && canvas.offsetHeight > 0; + }"; + + public string WaitForEsriView { get; set; } = @"() => { + try { + const elements = document.querySelectorAll('.esri-view'); + for (let el of elements) { + if (el.__view && el.__view.ready) { + return true; + } + } + } catch(e) {} + return false; + }"; + + public string CheckActiveFrameSegment { get; set; } = @"() => { + const segments = Array.from(document.querySelectorAll('[data-testid=""bom-scrub-segment""]')); + const activeSegment = segments.find(s => { + const style = window.getComputedStyle(s); + return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)'; + }); + return activeSegment && activeSegment.getAttribute('data-id') === '0'; + }"; + + public string WaitForMapContainer { get; set; } = @"() => { + const container = document.querySelector('.esri-view-surface'); + return container && container.offsetWidth > 0 && container.offsetHeight > 0; + }"; + + public string ExtractFrameInfo { get; set; } = @"() => { + const segments = Array.from(document.querySelectorAll('[data-testid=""bom-scrub-segment""]')); + const now = new Date(); + return segments.map((seg, index) => { + const ariaLabel = seg.getAttribute('aria-label') || ''; + let minutes = null; + + // Parse timestamp format: ""Wednesday 17 Dec, 11:05 pm"" or ""17 Dec, 11:05 pm"" + const timestampMatch = ariaLabel.match(/(?:[A-Za-z]+\s+)?(\d{1,2})\s+([A-Za-z]{3}),?\s+(\d{1,2}):(\d{2})\s+(am|pm)/i); + if (timestampMatch) { + try { + const day = parseInt(timestampMatch[1]); + const monthStr = timestampMatch[2]; + const hour12 = parseInt(timestampMatch[3]); + const minute = parseInt(timestampMatch[4]); + const ampm = timestampMatch[5].toLowerCase(); + + const monthNames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']; + const month = monthNames.indexOf(monthStr.toLowerCase()); + + if (month >= 0) { + let hour24 = hour12; + if (ampm === 'pm' && hour12 !== 12) hour24 += 12; + if (ampm === 'am' && hour12 === 12) hour24 = 0; + + const year = now.getFullYear(); + const frameTime = new Date(year, month, day, hour24, minute); + + // If parsed time is in future, assume it's from last year + if (frameTime > now) { + frameTime.setFullYear(year - 1); + } + + const diffMs = now - frameTime; + minutes = Math.round(diffMs / (1000 * 60)); + + // Validate reasonable range (0-2 hours) + if (minutes < 0 || minutes > 120) { + minutes = null; + } + } + } catch(e) { + // Parse error, leave minutes as null + } + } + + return { index: index, minutesAgo: minutes }; + }); + }"; + + public string CheckModalOverlay { get; set; } = @"() => { + const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); + if (bomOverlay && bomOverlay.style.display !== 'none') { + return true; + } + + const recaptchaSelectors = ['.g-recaptcha', '#g-recaptcha', '.rc-anchor-container']; + for (const selector of recaptchaSelectors) { + const el = document.querySelector(selector); + if (el) { + const style = window.getComputedStyle(el); + const rect = el.getBoundingClientRect(); + if (style.display !== 'none' && style.visibility !== 'hidden' && + rect.width > 200 && rect.height > 200) { + return true; + } + } + } + + const allForms = document.querySelectorAll('form'); + for (const form of allForms) { + const action = form.getAttribute('action') || ''; + const id = form.getAttribute('id') || ''; + if (action.includes('feedback') || id.includes('feedback')) { + const style = window.getComputedStyle(form); + const rect = form.getBoundingClientRect(); + if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { + const text = form.textContent || ''; + if (text.includes('reCAPTCHA') || text.includes('recaptcha') || text.includes('Tell us why')) { + return true; + } + } + } + } + + return false; + }"; + + public string CheckModalStillVisible { get; set; } = @"() => { + const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); + if (bomOverlay && bomOverlay.style.display !== 'none') return true; + + const allForms = document.querySelectorAll('form'); + for (const form of allForms) { + const action = form.getAttribute('action') || ''; + const id = form.getAttribute('id') || ''; + if (action.includes('feedback') || id.includes('feedback')) { + const style = window.getComputedStyle(form); + const rect = form.getBoundingClientRect(); + if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { + return true; + } + } + } + return false; + }"; + + public string GetViewportSize { get; set; } = @"() => JSON.stringify({ width: window.innerWidth, height: window.innerHeight })"; + + public string ExtractWeatherMetadata { get; set; } = @"() => { + const section = document.querySelector('section[data-testid=""weatherMetadata""]') || + document.querySelector('section[aria-label=""Last updated""]'); + if (!section) return null; + + const divs = section.querySelectorAll('div'); + return Array.from(divs).map(div => div.textContent.trim()).filter(text => text).join(' '); + }"; +} + diff --git a/Models/ScrapingSelectorsConfig.cs b/Models/ScrapingSelectorsConfig.cs new file mode 100644 index 0000000..1f1b178 --- /dev/null +++ b/Models/ScrapingSelectorsConfig.cs @@ -0,0 +1,26 @@ +namespace BomLocalService.Models; + +/// +/// Configuration for all selectors used in scraping +/// +public class ScrapingSelectorsConfig +{ + public SelectorConfig SearchButton { get; set; } = new(); + public SelectorConfig SearchInput { get; set; } = new(); + public SelectorConfig SearchResultsList { get; set; } = new(); + public SelectorConfig SearchResultItem { get; set; } = new(); + public SelectorConfig LocationName { get; set; } = new(); + public SelectorConfig LocationDescription { get; set; } = new(); + public SelectorConfig ResultsTitle { get; set; } = new(); + public SelectorConfig RadarLink { get; set; } = new(); + public SelectorConfig MapCanvas { get; set; } = new(); + public SelectorConfig MapContainer { get; set; } = new(); + public SelectorConfig PlayPauseButton { get; set; } = new(); + public SelectorConfig PlayPauseLabel { get; set; } = new(); + public SelectorConfig FrameSegment { get; set; } = new(); + public SelectorConfig StepForwardButton { get; set; } = new(); + public SelectorConfig TimeDisplayLabel { get; set; } = new(); + public SelectorConfig ModalOverlay { get; set; } = new(); + public SelectorConfig WeatherMetadata { get; set; } = new(); +} + diff --git a/Models/ScrapingStepConfig.cs b/Models/ScrapingStepConfig.cs new file mode 100644 index 0000000..a95d9a5 --- /dev/null +++ b/Models/ScrapingStepConfig.cs @@ -0,0 +1,18 @@ +namespace BomLocalService.Models; + +/// +/// Configuration for a single scraping step +/// +public class ScrapingStepConfig +{ + /// + /// Whether this step is enabled (can be disabled without removing from workflow) + /// + public bool Enabled { get; set; } = true; + + /// + /// Optional parameters specific to this step + /// + public Dictionary? Parameters { get; set; } +} + diff --git a/Models/ScrapingWorkflowConfig.cs b/Models/ScrapingWorkflowConfig.cs new file mode 100644 index 0000000..b3d95f3 --- /dev/null +++ b/Models/ScrapingWorkflowConfig.cs @@ -0,0 +1,11 @@ +namespace BomLocalService.Models; + +/// +/// Configuration for a scraping workflow +/// +public class ScrapingWorkflowConfig +{ + public string Description { get; set; } = string.Empty; + public Dictionary Steps { get; set; } = new(); +} + diff --git a/Models/SelectorConfig.cs b/Models/SelectorConfig.cs new file mode 100644 index 0000000..8db3b41 --- /dev/null +++ b/Models/SelectorConfig.cs @@ -0,0 +1,33 @@ +namespace BomLocalService.Models; + +/// +/// Configuration for a single selector with multiple fallback options +/// +public class SelectorConfig +{ + /// + /// Human-readable name for this selector (for logging/debugging) + /// + public string Name { get; set; } = string.Empty; + + /// + /// Array of CSS selectors to try in order (first match wins) + /// + public string[] Selectors { get; set; } = Array.Empty(); + + /// + /// Timeout in milliseconds when waiting for this element + /// + public int TimeoutMs { get; set; } = 5000; + + /// + /// Whether this element is required (throws if not found) + /// + public bool Required { get; set; } = true; + + /// + /// Custom error message if element is not found (when Required=true) + /// + public string? ErrorMessage { get; set; } +} + diff --git a/Models/TextPatternsConfig.cs b/Models/TextPatternsConfig.cs new file mode 100644 index 0000000..1ab9077 --- /dev/null +++ b/Models/TextPatternsConfig.cs @@ -0,0 +1,23 @@ +namespace BomLocalService.Models; + +/// +/// Text patterns and regex configurations for parsing page content +/// +public class TextPatternsConfig +{ + public string ResultsCountPattern { get; set; } = @"(\d+)\s+of\s+(\d+)"; + public string TimestampPattern { get; set; } = @"(?:[A-Za-z]+\s+)?\d{1,2}\s+[A-Za-z]{3},?\s+\d{1,2}:\d{2}\s+(?:am|pm)"; + public string ObservationTimePattern { get; set; } = @"Observations:\s*(\d+)\s*minutes?\s*ago"; + public string ForecastTimePattern { get; set; } = @"Forecast:\s*(\d+)\s*minutes?\s+ago"; + public string ForecastHourAgoPattern { get; set; } = @"Forecast:\s*an\s+hour\s+ago"; + public string WeatherStationPattern { get; set; } = @"at\s+([^,]+)\s+weather\s+station"; + public string DistancePattern { get; set; } = @"(\d+)\s*km\s+from"; + + public Dictionary ExpectedTexts { get; set; } = new() + { + ["PlayButtonLabel"] = "Play", + ["PauseButtonLabel"] = "Pause", + ["RadarLinkText"] = "Rain radar and weather map" + }; +} + diff --git a/Program.cs b/Program.cs index 29d5b9c..966ed4b 100644 --- a/Program.cs +++ b/Program.cs @@ -1,5 +1,12 @@ using BomLocalService.Services; using BomLocalService.Services.Interfaces; +using BomLocalService.Services.Scraping; +using BomLocalService.Services.Scraping.Steps.Navigation; +using BomLocalService.Services.Scraping.Steps.Search; +using BomLocalService.Services.Scraping.Steps.Map; +using BomLocalService.Services.Scraping.Steps.Metadata; +using BomLocalService.Services.Scraping.Steps.Capture; +using BomLocalService.Services.Scraping.Workflows; var builder = WebApplication.CreateBuilder(args); @@ -8,6 +15,7 @@ var builder = WebApplication.CreateBuilder(args); builder.Services.AddControllersWithViews(); builder.Services.AddEndpointsApiExplorer(); builder.Services.AddOpenApi(); +builder.Services.AddHealthChecks(); // Configure CORS - MUST be added before other services var corsOrigins = builder.Configuration.GetValue("Cors:AllowedOrigins", "*"); @@ -82,6 +90,33 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); + +// Register scraping step registry +builder.Services.AddSingleton(); + +// Register all scraping steps +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); + +// Register workflows +builder.Services.AddScoped(); +builder.Services.AddScoped(); + +// Register workflow factory +builder.Services.AddSingleton(); + +// Register scraping service (depends on workflow factory) builder.Services.AddSingleton(); // Register BOM Radar Service as singleton (orchestrator, depends on all above services) @@ -129,6 +164,27 @@ app.MapControllerRoute( // Map API controllers (with /api prefix) app.MapControllers(); +// Map health check endpoint for Docker health monitoring +app.MapHealthChecks("/api/health"); + +// Auto-register all scraping steps in the registry +var stepRegistry = app.Services.GetRequiredService(); +var stepTypes = typeof(IScrapingStep).Assembly.GetTypes() + .Where(t => typeof(IScrapingStep).IsAssignableFrom(t) && !t.IsInterface && !t.IsAbstract && !t.IsGenericType); +foreach (var stepType in stepTypes) +{ + try + { + var step = (IScrapingStep)ActivatorUtilities.CreateInstance(app.Services, stepType); + stepRegistry.RegisterStep(step); + } + catch (Exception ex) + { + var logger = app.Services.GetRequiredService>(); + logger.LogWarning(ex, "Failed to register step {StepType}", stepType.Name); + } +} + // Cleanup incomplete cache folders from previous crashes/restarts before starting services var cacheService = app.Services.GetRequiredService(); var deletedCount = cacheService.CleanupIncompleteCacheFolders(); diff --git a/README.md b/README.md index 4e451fd..1afc742 100644 --- a/README.md +++ b/README.md @@ -41,10 +41,25 @@ Built on ASP.NET Core 9.0, the service uses a service-oriented architecture with - **BomRadarService**: Main orchestrator that coordinates cache operations, browser automation, and data retrieval - **CacheService**: Manages file-based storage of radar screenshots and metadata in organized directory structures - **BrowserService**: Handles Playwright browser automation for headless browser sessions -- **ScrapingService**: Performs web scraping operations using the browser service to navigate and capture radar data from the BOM website +- **ScrapingService**: Coordinates web scraping workflows (simplified orchestrator) +- **SelectorService**: Finds page elements using configurable CSS selectors with fallback support - **TimeParsingService**: Parses and converts time formats from BOM data - **DebugService**: Provides debug functionality for troubleshooting +### Scraping Architecture +The scraping system uses a **workflow-based architecture** with configurable steps: + +- **Workflows**: Define fixed sequences of steps for different data types (e.g., `RadarScrapingWorkflow`, `TemperatureMapWorkflow`). Each workflow specifies its response type via generics (`IWorkflow`) +- **Steps**: Individual, testable units that perform specific actions (navigation, search, map interaction, capture). Steps declare prerequisites and validate page state before execution +- **Step Registry**: Manages and discovers available scraping steps +- **Workflow Factory**: Creates typed workflow instances based on configuration + +**Configuration-Driven Design:** +- **Selectors**: All CSS selectors are configurable via `appsettings.json` with fallback options, allowing adaptation to website changes without code modifications +- **JavaScript Templates**: JavaScript code for page evaluation is externalized in configuration, making it easy to update logic as the website evolves +- **Text Patterns**: Regex patterns for parsing page content are configurable, enabling quick adjustments to parsing logic +- **Workflow Steps**: Individual steps within workflows can be enabled/disabled via configuration, providing flexibility for testing and troubleshooting + ### Background Services - **CacheManagementService**: Periodically checks cache validity for all cached locations and triggers updates when data expires - **CacheCleanupService**: Removes cache files older than the configured retention period @@ -163,11 +178,23 @@ If you want to build locally, keep the `build:` section in `docker-compose.yml` docker-compose up -d ``` +**Mounting custom appsettings.json:** + +To use a custom configuration file, add it to the volumes section in `docker-compose.yml`: +```yaml +services: + bom-local-service: + volumes: + - ./cache:/app/cache + - ./appsettings.json:/app/appsettings.json:ro # Custom config +``` + This will: - Pull/build the image as configured - Start the service on port 8082 (configurable via `HOST_PORT`) - Mount the `./cache` directory for persistent storage -- Apply all environment variable configurations +- Mount custom `appsettings.json` if specified +- Apply all environment variable configurations (which override appsettings.json) To view logs: ```bash @@ -221,8 +248,50 @@ All configuration can be done via environment variables, which override the defa | Variable | Description | Default | Example | |----------|-------------|---------|---------| -| `SCREENSHOT__DYNAMICCONTENTWAITMS` | Milliseconds to wait for dynamic content to load | `2000` | `3000` | -| `SCREENSHOT__TILERENDERWAITMS` | Milliseconds to wait for map tiles to render | `5000` | `7000` | +| `SCREENSHOT__DYNAMICCONTENTWAITMS` | Milliseconds to wait for dynamic content to load | `1500` | `2000` | +| `SCREENSHOT__TILERENDERWAITMS` | Milliseconds to wait for map tiles to render | `3000` | `5000` | +| `SCREENSHOT__CROP__X` | X offset in pixels for screenshot cropping | `250` | `300` | +| `SCREENSHOT__CROP__Y` | Y offset in pixels for screenshot cropping | `0` | `50` | +| `SCREENSHOT__CROP__RIGHTOFFSET` | Right offset in pixels for screenshot cropping | `250` | `300` | +| `SCREENSHOT__CROP__HEIGHT` | Height in pixels for screenshot cropping (null = full height) | `null` | `800` | + +#### Scraping Configuration + +The scraping system is highly configurable through `appsettings.json`. Most scraping settings (selectors, JavaScript templates, text patterns, workflow steps) are configured in `appsettings.json`, but can be overridden in Docker deployments. + +**Option 1: Mount Custom appsettings.json (Recommended for Docker)** + +Mount a custom `appsettings.json` file as a volume: + +```bash +docker run -d \ + --name bom-local-service \ + -p 8082:8080 \ + -v $(pwd)/cache:/app/cache \ + -v $(pwd)/appsettings.json:/app/appsettings.json:ro \ + --shm-size=1gb \ + --ipc=host \ + ghcr.io/alexhopeoconnor/bom-local-service:latest +``` + +Or in `docker-compose.yml`: +```yaml +services: + bom-local-service: + volumes: + - ./cache:/app/cache + - ./appsettings.json:/app/appsettings.json:ro # Add this line +``` + +**Option 2: Environment Variables (Simple Overrides)** + +For quick overrides of commonly needed settings: + +| Variable | Description | Default | Example | +|----------|-------------|---------|---------| +| `SCRAPING__BASEURL` | Base URL for the BOM website | `https://www.bom.gov.au/` | `https://www.bom.gov.au/` | + +**Note**: Complex configurations (selectors, JavaScript templates, text patterns, workflow steps) are best managed via a mounted `appsettings.json` file. See the [Configuration File](#configuration-file) section below for the complete structure. #### Debug Configuration @@ -306,6 +375,126 @@ Then run: docker-compose up -d ``` +#### Custom appsettings.json (For Selector/Scraping Configuration) + +If you need to customize selectors, JavaScript templates, or workflow steps: + +1. **Copy the default appsettings.json** from the repository +2. **Edit the sections you need** (e.g., `Scraping:Selectors`) +3. **Mount it as a volume**: + +```bash +docker run -d \ + --name bom-local-service \ + -p 8082:8080 \ + -v $(pwd)/cache:/app/cache \ + -v $(pwd)/appsettings.json:/app/appsettings.json:ro \ + --shm-size=1gb \ + --ipc=host \ + ghcr.io/alexhopeoconnor/bom-local-service:latest +``` + +Or in `docker-compose.yml`: +```yaml +services: + bom-local-service: + volumes: + - ./cache:/app/cache + - ./appsettings.json:/app/appsettings.json:ro +``` + +**Note**: Environment variables still override values in the mounted `appsettings.json`, so you can use env vars for simple overrides and the mounted file for complex configurations. + +### Configuration File + +For advanced scraping configuration, edit `appsettings.json` directly. When using Docker, you can mount a custom `appsettings.json` file as a volume (see [Scraping Configuration](#scraping-configuration) above). + +The scraping system supports extensive configuration: + +#### Scraping Selectors + +All CSS selectors used to find page elements are configurable with fallback options: + +```json +{ + "Scraping": { + "Selectors": { + "SearchButton": { + "Name": "Search Button", + "Selectors": [ + "button[data-testid='searchLabel']", + "button[aria-label='Search for a location']", + "button.search-location__trigger-button" + ], + "TimeoutMs": 10000, + "Required": true, + "ErrorMessage": "Could not find search button" + } + } + } +} +``` + +#### JavaScript Templates + +JavaScript code used for page evaluation is externalized and configurable: + +```json +{ + "Scraping": { + "JavaScriptTemplates": { + "WaitForSearchResults": "() => { /* template code */ }", + "ExtractSearchResults": "() => { /* template code */ }" + } + } +} +``` + +#### Text Patterns + +Regex patterns for parsing page content are configurable: + +```json +{ + "Scraping": { + "TextPatterns": { + "ResultsCountPattern": "(\\d+)\\s+of\\s+(\\d+)", + "TimestampPattern": "(?:[A-Za-z]+\\s+)?\\d{1,2}\\s+[A-Za-z]{3},?\\s+\\d{1,2}:\\d{2}\\s+(?:am|pm)", + "TimestampPattern": "(?:[A-Za-z]+\\s+)?\\d{1,2}\\s+[A-Za-z]{3},?\\s+\\d{1,2}:\\d{2}\\s+(?:am|pm)" + } + } +} +``` + +#### Workflow Steps + +Individual workflow steps can be enabled/disabled and configured: + +```json +{ + "Scraping": { + "Workflows": { + "RadarScraping": { + "Description": "Scrapes radar images for a location", + "Steps": { + "NavigateHomepage": { "Enabled": true }, + "ClickSearchButton": { "Enabled": true }, + "CaptureFrames": { + "Enabled": true, + "Parameters": { + "FrameCount": 7, + "WaitBetweenFrames": 5000 + } + } + } + } + } + } +} +``` + +**Note**: Step order is fixed within workflows due to dependencies. Steps can be disabled but not reordered. See `appsettings.json` for the complete configuration structure. + ## API Documentation The service provides RESTful API endpoints for accessing radar data and managing the cache. @@ -711,7 +900,8 @@ The service uses a **metrics-based estimation system** to provide accurate estim 2. **Metrics Collection**: After each successful cache update, the service records: - Total duration of the update - - Duration of each phase + - Duration of each phase (Initializing, CapturingFrames, Saving) + - Duration of each individual scraping step (NavigateHomepage, ClickSearchButton, etc.) - Frame-level progress during capture 3. **Estimation Strategy**: @@ -729,7 +919,7 @@ The service uses a **metrics-based estimation system** to provide accurate estim **First Update (No Metrics)**: - Uses calculated estimate based on `Screenshot:DynamicContentWaitMs`, `Screenshot:TileRenderWaitMs`, and frame count -- Example: ~120 seconds for 7 frames with default settings +- Example: ~100 seconds for 7 frames with default settings (optimized wait times) **Subsequent Updates (With Metrics)**: - Uses median duration from historical data @@ -998,6 +1188,25 @@ fi - **Check cache status**: Use `/api/cache/{suburb}/{state}/range` to verify cache exists - **Browser automation issues**: Check logs for Playwright errors +### Scraping Failures + +If scraping fails (e.g., "Could not find element"), the BOM website structure may have changed: + +- **Check debug screenshots**: Enable `DEBUG__ENABLED=true` to see what the browser sees at each step +- **Update selectors**: + - **Docker**: Mount a custom `appsettings.json` with updated selectors (see [Scraping Configuration](#scraping-configuration)) + - **Local**: Edit `appsettings.json` under `Scraping:Selectors` to add new CSS selectors as fallbacks +- **Check step logs**: Each step logs its execution - look for which step failed +- **Selector fallbacks**: The system tries multiple selectors in order, so add new selectors to the existing arrays +- **Workflow steps**: Individual steps can be disabled via `Scraping:Workflows:RadarScraping:Steps:{StepName}:Enabled: false` if needed temporarily + +**Quick Fix for Docker Users:** + +1. Copy the default `appsettings.json` from the repository +2. Edit the selectors that are failing +3. Mount it as a volume: `-v $(pwd)/appsettings.json:/app/appsettings.json:ro` +4. Restart the container + ### Playwright Resource Usage Playwright browsers (Chromium) can consume significant CPU and memory: @@ -1031,6 +1240,25 @@ docker run --cpus="1.0" ... - **Increase cleanup frequency**: Lower `CACHECLEANUP__INTERVALHOURS` to clean up more often - **Limit locations**: The service automatically manages all cached locations; reduce the number of locations being cached to lower resource usage +### Performance Monitoring + +The service logs detailed performance metrics for each scraping workflow: + +**Step-Level Timing**: +- Each step logs its duration and compares it to historical averages +- Example: `Step WaitForMapReady completed in 45.30s (avg: 43.76s)` +- Steps that are >50% slower than average trigger warnings: `⚠️ Step WaitForMapReady took significantly longer than average: 75.45s (avg: 50.30s, +25.15s, +50.0% slower)` + +**Workflow-Level Timing**: +- Complete workflow duration is logged with step breakdown +- Example: `Workflow RadarScraping completed in 144.40s. Step breakdown: NavigateHomepage=4.96s, ClickSearchButton=2.70s, ...` +- Workflows that are >30% slower than average trigger warnings: `⚠️ Workflow RadarScraping took significantly longer than average: 189.45s (avg: 145.67s, +43.78s, +30.0% slower)` + +**Metrics Storage**: +- Step and phase durations are stored in memory (last 20 samples) +- Used for performance estimation and identifying bottlenecks +- Metrics improve over time as more updates complete + ## Development ### Building Locally @@ -1049,7 +1277,45 @@ Enable debug mode to save intermediate screenshots during data capture: docker run -e DEBUG__ENABLED=true -e DEBUG__WAITMS=5000 ... ``` -Debug screenshots are saved in `{CACHEDIRECTORY}/debug/`. +Debug screenshots are saved in `{CACHEDIRECTORY}/debug/`. Each scraping step saves a screenshot, HTML snapshot, and logs, making it easy to diagnose issues. + +### Extending the Scraping System + +The workflow-based architecture makes it easy to extend the scraping system: + +**Adding a New Workflow**: + +1. Create a new workflow class in `Services/Scraping/Workflows/` implementing `IWorkflow` where `TResponse` is your response type +2. Define the step sequence (can reuse existing steps) +3. Register the workflow in `WorkflowFactory` +4. Add workflow configuration to `appsettings.json` + +**Adding a New Step**: + +1. Create a step class inheriting from `BaseScrapingStep` +2. Implement `Name`, `Prerequisites`, `CanExecute`, and `ExecuteAsync` +3. The step will be auto-registered on startup +4. Add the step to a workflow's `StepNames` array + +**Updating Selectors**: + +1. Edit `appsettings.json` under `Scraping:Selectors` +2. Add new CSS selectors to the `Selectors` array (tried in order) +3. Adjust `TimeoutMs` if needed +4. No code changes required + +**Updating JavaScript Templates**: + +1. Edit `appsettings.json` under `Scraping:JavaScriptTemplates` +2. Update the template code as needed +3. No code changes required + +**Updating Text Patterns**: + +1. Edit `appsettings.json` under `Scraping:TextPatterns` +2. Update regex patterns as needed (e.g., `TimestampPattern`) +3. The `TimestampPattern` supports parsing timestamps like "Wednesday 17 Dec, 11:05 pm" when the BOM website changes format +4. No code changes required ## License diff --git a/Services/CacheService.cs b/Services/CacheService.cs index 54b2cc2..8f8a944 100644 --- a/Services/CacheService.cs +++ b/Services/CacheService.cs @@ -18,6 +18,7 @@ public class CacheService : ICacheService private readonly ConcurrentDictionary _updateProgress = new(); private readonly ConcurrentQueue _recentTotalDurations = new(); // Overall durations in seconds private readonly ConcurrentDictionary> _phaseDurations = new(); // Phase -> durations + private readonly ConcurrentDictionary> _stepDurations = new(); // Step name -> durations private readonly object _metricsLock = new(); private const int MaxSamples = 20; @@ -589,7 +590,7 @@ public class CacheService : ICacheService } } - _logger.LogDebug("Cache update completed in {Duration:F1} seconds for {Location}", totalDuration, locationKey); + _logger.LogInformation("Cache update completed in {Duration:F1} seconds for {Location}", totalDuration, locationKey); } } @@ -665,7 +666,7 @@ public class CacheService : ICacheService /// /// Gets the average total duration of cache updates from recent metrics. /// - private double GetAverageTotalDuration() + public double GetAverageTotalDuration() { lock (_metricsLock) { @@ -707,6 +708,59 @@ public class CacheService : ICacheService } } + /// + /// Records step completion timing for metrics tracking. + /// + public void RecordStepCompletion(string stepName, double durationSeconds) + { + lock (_metricsLock) + { + var durations = _stepDurations.GetOrAdd(stepName, _ => new ConcurrentQueue()); + durations.Enqueue(durationSeconds); + + while (durations.Count > MaxSamples) + { + durations.TryDequeue(out _); + } + } + } + + /// + /// Gets the average duration for a specific step from historical data. + /// + public double GetAverageStepDuration(string stepName) + { + lock (_metricsLock) + { + if (!_stepDurations.TryGetValue(stepName, out var durations) || durations.Count == 0) + { + return 0; + } + var durationsArray = durations.ToArray(); + return durationsArray.Average(); + } + } + + /// + /// Gets step performance metrics for debugging/logging. + /// + public Dictionary GetStepMetrics() + { + lock (_metricsLock) + { + var metrics = new Dictionary(); + foreach (var kvp in _stepDurations) + { + if (kvp.Value.Count > 0) + { + var durations = kvp.Value.ToArray(); + metrics[kvp.Key] = durations.Average(); + } + } + return metrics; + } + } + /// /// Gets the locationKey from a cacheFolderPath by parsing the folder name. /// diff --git a/Services/DebugService.cs b/Services/DebugService.cs index f3af430..238a434 100644 --- a/Services/DebugService.cs +++ b/Services/DebugService.cs @@ -90,18 +90,22 @@ namespace BomLocalService.Services; await File.WriteAllTextAsync(htmlPath, htmlContent, cancellationToken); // Save console messages if provided + // Create a snapshot to avoid collection modification during enumeration if (consoleMessages != null && consoleMessages.Count > 0) { var consolePath = Path.Combine(stepFolder, "console.log"); - var consoleText = string.Join("\n", consoleMessages.Select(msg => $"[{msg.timestamp:HH:mm:ss.fff}] [{msg.type}] {msg.text}")); + var consoleSnapshot = consoleMessages.ToList(); // Create snapshot + var consoleText = string.Join("\n", consoleSnapshot.Select(msg => $"[{msg.timestamp:HH:mm:ss.fff}] [{msg.type}] {msg.text}")); await File.WriteAllTextAsync(consolePath, consoleText, cancellationToken); } // Save network requests summary if provided + // Create a snapshot to avoid collection modification during enumeration if (networkRequests != null && networkRequests.Count > 0) { var networkPath = Path.Combine(stepFolder, "network.log"); - var networkText = string.Join("\n", networkRequests.Select(req => + var networkSnapshot = networkRequests.ToList(); // Create snapshot + var networkText = string.Join("\n", networkSnapshot.Select(req => $"[{req.timestamp:HH:mm:ss.fff}] {req.method} {req.url} -> {req.status?.ToString() ?? "pending"} ({req.resourceType})" )); await File.WriteAllTextAsync(networkPath, networkText, cancellationToken); @@ -148,18 +152,22 @@ namespace BomLocalService.Services; } // Save console messages if provided + // Create a snapshot to avoid collection modification during enumeration if (consoleMessages != null && consoleMessages.Count > 0) { var consolePath = Path.Combine(errorFolder, "console.log"); - var consoleText = string.Join("\n", consoleMessages.Select(msg => $"[{msg.timestamp:HH:mm:ss.fff}] [{msg.type}] {msg.text}")); + var consoleSnapshot = consoleMessages.ToList(); // Create snapshot + var consoleText = string.Join("\n", consoleSnapshot.Select(msg => $"[{msg.timestamp:HH:mm:ss.fff}] [{msg.type}] {msg.text}")); await File.WriteAllTextAsync(consolePath, consoleText, cancellationToken); } // Save network requests summary if provided + // Create a snapshot to avoid collection modification during enumeration if (networkRequests != null && networkRequests.Count > 0) { var networkPath = Path.Combine(errorFolder, "network.log"); - var networkText = string.Join("\n", networkRequests.Select(req => + var networkSnapshot = networkRequests.ToList(); // Create snapshot + var networkText = string.Join("\n", networkSnapshot.Select(req => $"[{req.timestamp:HH:mm:ss.fff}] {req.method} {req.url} -> {req.status?.ToString() ?? "pending"} ({req.resourceType})" )); await File.WriteAllTextAsync(networkPath, networkText, cancellationToken); diff --git a/Services/Interfaces/ICacheService.cs b/Services/Interfaces/ICacheService.cs index 4984026..5eb381f 100644 --- a/Services/Interfaces/ICacheService.cs +++ b/Services/Interfaces/ICacheService.cs @@ -213,5 +213,32 @@ public interface ICacheService /// The location key (suburb_state) /// Estimated remaining seconds, or 0 if not updating or no metrics int GetEstimatedRemainingSeconds(string locationKey); + + /// + /// Records step completion timing for metrics tracking. + /// + /// The name of the step + /// The duration of the step in seconds + void RecordStepCompletion(string stepName, double durationSeconds); + + /// + /// Gets the average duration for a specific step from historical data. + /// + /// The name of the step + /// Average duration in seconds, or 0 if no data available + double GetAverageStepDuration(string stepName); + + /// + /// Gets step performance metrics for debugging/logging. + /// + /// Dictionary mapping step names to their average durations + Dictionary GetStepMetrics(); + + /// + /// Gets the average total duration of cache updates from recent metrics. + /// Uses median for robustness against outliers. + /// + /// Average total duration in seconds, or 0 if no data available + double GetAverageTotalDuration(); } diff --git a/Services/Interfaces/ISelectorService.cs b/Services/Interfaces/ISelectorService.cs new file mode 100644 index 0000000..420c3fa --- /dev/null +++ b/Services/Interfaces/ISelectorService.cs @@ -0,0 +1,21 @@ +using BomLocalService.Models; +using Microsoft.Playwright; + +namespace BomLocalService.Services.Interfaces; + +/// +/// Service for finding page elements using configured selectors +/// +public interface ISelectorService +{ + /// + /// Finds an element using the configured selectors, trying each in order until one is found + /// + Task FindElementAsync(IPage page, SelectorConfig config, CancellationToken cancellationToken = default); + + /// + /// Gets a locator for the first selector (assumes element exists) + /// + ILocator GetLocator(IPage page, SelectorConfig config); +} + diff --git a/Services/Scraping/IScrapingStep.cs b/Services/Scraping/IScrapingStep.cs new file mode 100644 index 0000000..15fed03 --- /dev/null +++ b/Services/Scraping/IScrapingStep.cs @@ -0,0 +1,28 @@ +namespace BomLocalService.Services.Scraping; + +/// +/// Interface for a single scraping step +/// +public interface IScrapingStep +{ + /// + /// Unique name of the step + /// + string Name { get; } + + /// + /// Names of steps that must complete before this step can execute + /// + string[] Prerequisites { get; } + + /// + /// Checks if the step can execute in the current page state + /// + bool CanExecute(ScrapingContext context); + + /// + /// Executes the step + /// + Task ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken); +} + diff --git a/Services/Scraping/IScrapingStepRegistry.cs b/Services/Scraping/IScrapingStepRegistry.cs new file mode 100644 index 0000000..9be02d0 --- /dev/null +++ b/Services/Scraping/IScrapingStepRegistry.cs @@ -0,0 +1,12 @@ +namespace BomLocalService.Services.Scraping; + +/// +/// Registry for managing scraping steps +/// +public interface IScrapingStepRegistry +{ + void RegisterStep(IScrapingStep step); + IScrapingStep? GetStep(string name); + IEnumerable GetAllSteps(); +} + diff --git a/Services/Scraping/IWorkflow.cs b/Services/Scraping/IWorkflow.cs new file mode 100644 index 0000000..48be216 --- /dev/null +++ b/Services/Scraping/IWorkflow.cs @@ -0,0 +1,14 @@ +namespace BomLocalService.Services.Scraping; + +/// +/// Interface for a scraping workflow +/// +/// The response type returned by this workflow +public interface IWorkflow +{ + string Name { get; } + string Description { get; } + string[] StepNames { get; } // Fixed order - cannot be changed + Task ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken); +} + diff --git a/Services/Scraping/IWorkflowFactory.cs b/Services/Scraping/IWorkflowFactory.cs new file mode 100644 index 0000000..1e7a2fa --- /dev/null +++ b/Services/Scraping/IWorkflowFactory.cs @@ -0,0 +1,10 @@ +namespace BomLocalService.Services.Scraping; + +/// +/// Factory for creating workflows +/// +public interface IWorkflowFactory +{ + IWorkflow GetWorkflow(string name); +} + diff --git a/Services/Scraping/PageState.cs b/Services/Scraping/PageState.cs new file mode 100644 index 0000000..45678f5 --- /dev/null +++ b/Services/Scraping/PageState.cs @@ -0,0 +1,18 @@ +namespace BomLocalService.Services.Scraping; + +/// +/// Represents the current state of the page during scraping +/// +public enum PageState +{ + Initial, + HomepageLoaded, + SearchModalOpen, + SearchResultsVisible, + ForecastPageLoaded, + RadarPageLoaded, + MapReady, + SlideshowPaused, + Frame0Selected +} + diff --git a/Services/Scraping/ScrapingContext.cs b/Services/Scraping/ScrapingContext.cs new file mode 100644 index 0000000..3f55e94 --- /dev/null +++ b/Services/Scraping/ScrapingContext.cs @@ -0,0 +1,42 @@ +using BomLocalService.Models; +using Microsoft.Playwright; + +namespace BomLocalService.Services.Scraping; + +/// +/// Context shared between scraping steps +/// +public class ScrapingContext +{ + public IPage Page { get; set; } = null!; + public string Suburb { get; set; } = string.Empty; + public string State { get; set; } = string.Empty; + public string CacheFolderPath { get; set; } = string.Empty; + public string DebugFolder { get; set; } = string.Empty; + public List<(string type, string text, DateTime timestamp)> ConsoleMessages { get; set; } = new(); + public List<(string method, string url, int? status, string resourceType, DateTime timestamp)> NetworkRequests { get; set; } = new(); + + // Page state tracking + public PageState CurrentState { get; set; } = PageState.Initial; + public HashSet CompletedSteps { get; set; } = new(); + + // Shared state between steps + public ILocator? SearchButton { get; set; } + public ILocator? SearchInput { get; set; } + public List<(string name, string desc, string fullText)>? SearchResults { get; set; } + public int? SelectedResultIndex { get; set; } + public ILocator? RadarLink { get; set; } + public ILocator? MapContainer { get; set; } + public Clip? MapBoundingBox { get; set; } + public LastUpdatedInfo? LastUpdatedInfo { get; set; } + public List<(int index, int minutesAgo)>? FrameInfo { get; set; } + public List Frames { get; set; } = new(); + + // State validation helpers + public bool IsHomepageLoaded => CurrentState >= PageState.HomepageLoaded; + public bool IsSearchModalOpen => CurrentState >= PageState.SearchModalOpen; + public bool IsForecastPageLoaded => CurrentState >= PageState.ForecastPageLoaded; + public bool IsRadarPageLoaded => CurrentState >= PageState.RadarPageLoaded; + public bool IsMapReady => CurrentState >= PageState.MapReady; +} + diff --git a/Services/Scraping/ScrapingStepRegistry.cs b/Services/Scraping/ScrapingStepRegistry.cs new file mode 100644 index 0000000..92de591 --- /dev/null +++ b/Services/Scraping/ScrapingStepRegistry.cs @@ -0,0 +1,33 @@ +namespace BomLocalService.Services.Scraping; + +public class ScrapingStepRegistry : IScrapingStepRegistry +{ + private readonly Dictionary _steps = new(); + private readonly ILogger _logger; + + public ScrapingStepRegistry(ILogger logger) + { + _logger = logger; + } + + public void RegisterStep(IScrapingStep step) + { + if (_steps.ContainsKey(step.Name)) + { + _logger.LogWarning("Step {Name} is already registered, overwriting", step.Name); + } + _steps[step.Name] = step; + _logger.LogDebug("Registered step: {Name}", step.Name); + } + + public IScrapingStep? GetStep(string name) + { + return _steps.TryGetValue(name, out var step) ? step : null; + } + + public IEnumerable GetAllSteps() + { + return _steps.Values; + } +} + diff --git a/Services/Scraping/ScrapingStepResult.cs b/Services/Scraping/ScrapingStepResult.cs new file mode 100644 index 0000000..43cd663 --- /dev/null +++ b/Services/Scraping/ScrapingStepResult.cs @@ -0,0 +1,15 @@ +namespace BomLocalService.Services.Scraping; + +/// +/// Result of executing a scraping step +/// +public class ScrapingStepResult +{ + public bool Success { get; set; } + public string? ErrorMessage { get; set; } + public Dictionary? Data { get; set; } + + public static ScrapingStepResult Successful() => new() { Success = true }; + public static ScrapingStepResult Failed(string errorMessage) => new() { Success = false, ErrorMessage = errorMessage }; +} + diff --git a/Services/Scraping/Steps/BaseScrapingStep.cs b/Services/Scraping/Steps/BaseScrapingStep.cs new file mode 100644 index 0000000..4ddd3f1 --- /dev/null +++ b/Services/Scraping/Steps/BaseScrapingStep.cs @@ -0,0 +1,64 @@ +using BomLocalService.Models; +using BomLocalService.Services.Interfaces; +using Microsoft.Playwright; + +namespace BomLocalService.Services.Scraping.Steps; + +/// +/// Base class for scraping steps with common functionality +/// +public abstract class BaseScrapingStep : IScrapingStep +{ + protected readonly ILogger Logger; + protected readonly ISelectorService SelectorService; + protected readonly IDebugService DebugService; + protected readonly IConfiguration Configuration; + protected readonly ScrapingSelectorsConfig Selectors; + protected readonly JavaScriptTemplatesConfig JavaScriptTemplates; + protected readonly TextPatternsConfig TextPatterns; + + protected BaseScrapingStep( + ILogger logger, + ISelectorService selectorService, + IDebugService debugService, + IConfiguration configuration) + { + Logger = logger; + SelectorService = selectorService; + DebugService = debugService; + Configuration = configuration; + + Selectors = configuration.GetSection("Scraping:Selectors").Get() ?? new(); + JavaScriptTemplates = configuration.GetSection("Scraping:JavaScriptTemplates").Get() ?? new(); + TextPatterns = configuration.GetSection("Scraping:TextPatterns").Get() ?? new(); + } + + public abstract string Name { get; } + public abstract string[] Prerequisites { get; } + public abstract bool CanExecute(ScrapingContext context); + public abstract Task ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken); + + protected async Task SaveDebugAsync(ScrapingContext context, int stepNumber, string stepName, CancellationToken cancellationToken) + { + await DebugService.SaveStepDebugAsync( + context.DebugFolder, + stepNumber, + stepName, + context.Page, + context.ConsoleMessages, + context.NetworkRequests, + cancellationToken); + } + + protected async Task SaveErrorDebugAsync(ScrapingContext context, string errorMessage, CancellationToken cancellationToken) + { + await DebugService.SaveErrorDebugAsync( + context.DebugFolder, + errorMessage, + context.Page, + context.ConsoleMessages, + context.NetworkRequests, + cancellationToken); + } +} + diff --git a/Services/Scraping/Steps/Capture/CaptureFramesStep.cs b/Services/Scraping/Steps/Capture/CaptureFramesStep.cs new file mode 100644 index 0000000..68a8ffe --- /dev/null +++ b/Services/Scraping/Steps/Capture/CaptureFramesStep.cs @@ -0,0 +1,489 @@ +using BomLocalService.Models; +using BomLocalService.Services.Interfaces; +using BomLocalService.Services.Scraping; +using BomLocalService.Utilities; +using Microsoft.Playwright; +using System.Text.RegularExpressions; + +namespace BomLocalService.Services.Scraping.Steps.Capture; + +public class CaptureFramesStep : BaseScrapingStep +{ + private readonly ICacheService _cacheService; + private readonly int _tileRenderWaitMs; + private readonly ScreenshotCropConfig _cropConfig; + + public override string Name => "CaptureFrames"; + public override string[] Prerequisites => new[] { "ExtractMetadata", "CalculateMapBounds" }; + + public CaptureFramesStep( + ILogger logger, + ISelectorService selectorService, + IDebugService debugService, + IConfiguration configuration, + ICacheService cacheService) + : base(logger, selectorService, debugService, configuration) + { + _cacheService = cacheService; + _tileRenderWaitMs = configuration.GetValue("Screenshot:TileRenderWaitMs", 5000); + + var cropSection = configuration.GetSection("Screenshot:Crop"); + _cropConfig = new ScreenshotCropConfig + { + X = cropSection.GetValue("X", 0), + Y = cropSection.GetValue("Y", 0), + RightOffset = cropSection.GetValue("RightOffset", 0), + Height = cropSection.GetValue("Height") + }; + } + + public override bool CanExecute(ScrapingContext context) + { + return context.MapBoundingBox != null && context.MapContainer != null; + } + + public override async Task ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken) + { + try + { + Directory.CreateDirectory(context.CacheFolderPath); + Logger.LogInformation("Step {Step}: Using cache folder: {Path}", Name, context.CacheFolderPath); + + var frameCount = CacheHelper.GetFrameCountForDataType(Configuration, CachedDataType.Radar); + + _cacheService.RecordUpdateProgressByFolder(context.CacheFolderPath, CacheUpdatePhase.CapturingFrames, 0, frameCount); + + var frames = new List(); + var stepForwardButton = SelectorService.GetLocator(context.Page, Selectors.StepForwardButton); + int? previousMinutesAgo = null; + + for (int frameIndex = 0; frameIndex < frameCount; frameIndex++) + { + Logger.LogInformation("Step {Step}: Capturing frame {FrameIndex} of {FrameCount}", Name, frameIndex, frameCount); + + await context.Page.WaitForTimeoutAsync(_tileRenderWaitMs); + + // Small wait to ensure display label is stable after frame change/selection + // This is especially important for frame 0 which was just selected in ResetToFirstFrame + await context.Page.WaitForTimeoutAsync(300); + + // Try extracting with a retry in case the label is still updating + var minutesAgo = await ExtractMinutesAgoFromDisplayAsync(context.Page); + if (minutesAgo == null) + { + // Retry once after a short wait in case label was updating + await context.Page.WaitForTimeoutAsync(200); + minutesAgo = await ExtractMinutesAgoFromDisplayAsync(context.Page); + } + if (minutesAgo == null && context.FrameInfo != null && frameIndex < context.FrameInfo.Count) + { + var (_, defaultMinutesAgo) = context.FrameInfo[frameIndex]; + minutesAgo = defaultMinutesAgo; + Logger.LogWarning("Step {Step}: Failed to extract minutes from display label for frame {FrameIndex}, using default: {MinutesAgo}", Name, frameIndex, minutesAgo); + } + + if (frameIndex > 0 && previousMinutesAgo.HasValue && minutesAgo == previousMinutesAgo.Value) + { + Logger.LogWarning("Step {Step}: Frame {FrameIndex} has same minutesAgo ({MinutesAgo}) as previous frame. Waiting for display to update...", Name, frameIndex, minutesAgo); + await WaitForDisplayLabelToChangeAsync(context.Page, previousMinutesAgo.Value); + minutesAgo = await ExtractMinutesAgoFromDisplayAsync(context.Page); + if (minutesAgo == null || minutesAgo == previousMinutesAgo.Value) + { + if (context.FrameInfo != null && frameIndex < context.FrameInfo.Count) + { + var (_, defaultMinutesAgo) = context.FrameInfo[frameIndex]; + minutesAgo = defaultMinutesAgo; + Logger.LogWarning("Step {Step}: Display label did not update for frame {FrameIndex}, using calculated default: {MinutesAgo}", Name, frameIndex, minutesAgo); + } + } + } + + var radarFolder = FilePathHelper.GetDataTypeFolderPath(context.CacheFolderPath, CachedDataType.Radar); + if (!Directory.Exists(radarFolder)) + { + Directory.CreateDirectory(radarFolder); + } + + var framePath = FilePathHelper.GetFrameFilePath(context.CacheFolderPath, CachedDataType.Radar, frameIndex); + await CaptureMapScreenshotAsync(context.Page, context.MapContainer!, framePath, context.MapBoundingBox!); + + frames.Add(new RadarFrame + { + FrameIndex = frameIndex, + ImagePath = framePath, + MinutesAgo = minutesAgo ?? 0 + }); + + previousMinutesAgo = minutesAgo; + + Logger.LogInformation("Step {Step}: Frame {FrameIndex} saved: {Path} ({MinutesAgo} minutes ago)", + Name, frameIndex, framePath, minutesAgo ?? 0); + + _cacheService.RecordUpdateProgressByFolder(context.CacheFolderPath, CacheUpdatePhase.CapturingFrames, frameIndex + 1, frameCount); + + await SaveDebugAsync(context, 15 + frameIndex, $"frame_{frameIndex}_captured", cancellationToken); + + if (frameIndex < frameCount - 1) + { + await DismissModalOverlaysAsync(context.Page); + + var currentMinutesAgo = await ExtractMinutesAgoFromDisplayAsync(context.Page); + + await stepForwardButton.ClickAsync(new LocatorClickOptions { Force = true }); + + if (currentMinutesAgo.HasValue) + { + await WaitForDisplayLabelToChangeAsync(context.Page, currentMinutesAgo.Value); + } + else + { + await context.Page.WaitForTimeoutAsync(500); + } + } + } + + Logger.LogInformation("Step {Step}: All {FrameCount} frames captured successfully", Name, frameCount); + + _cacheService.RecordUpdateProgressByFolder(context.CacheFolderPath, CacheUpdatePhase.Saving); + + if (context.LastUpdatedInfo != null) + { + await _cacheService.SaveMetadataAsync(context.CacheFolderPath, context.LastUpdatedInfo, cancellationToken); + } + await _cacheService.SaveFramesMetadataAsync(context.CacheFolderPath, CachedDataType.Radar, frames, cancellationToken); + + context.Frames = frames; + + return ScrapingStepResult.Successful(); + } + catch (Exception ex) + { + Logger.LogError(ex, "Step {Step} failed", Name); + await SaveErrorDebugAsync(context, $"Failed to capture frames: {ex.Message}", cancellationToken); + return ScrapingStepResult.Failed($"Failed to capture frames: {ex.Message}"); + } + } + + private async Task ExtractMinutesAgoFromDisplayAsync(IPage page) + { + try + { + var timeLabelLocator = SelectorService.GetLocator(page, Selectors.TimeDisplayLabel); + var timeLabel = await timeLabelLocator.TextContentAsync(); + if (string.IsNullOrEmpty(timeLabel)) + { + Logger.LogDebug("Time display label is empty"); + return null; + } + + var trimmedLabel = timeLabel.Trim(); + Logger.LogInformation("Extracting minutes from display label: '{Label}'", trimmedLabel); + + // Parse timestamp format (current BOM website format): "Wednesday 17 Dec, 11:05 pm" or "17 Dec, 11:05 pm" + Logger.LogInformation("Trying timestamp pattern: '{Pattern}'", TextPatterns.TimestampPattern); + var timestampMatch = Regex.Match(trimmedLabel, TextPatterns.TimestampPattern, RegexOptions.IgnoreCase); + if (timestampMatch.Success) + { + var timestampStr = timestampMatch.Groups[0].Value; + Logger.LogInformation("Matched timestamp pattern: '{Timestamp}' from label: '{Label}'", timestampStr, trimmedLabel); + if (TryParseTimestamp(timestampStr, out var timestamp)) + { + var minutesAgo = (int)(DateTime.UtcNow - timestamp).TotalMinutes; + Logger.LogInformation("Parsed timestamp: {Timestamp} UTC, calculated minutes ago: {Minutes}", timestamp, minutesAgo); + if (minutesAgo >= 0 && minutesAgo <= 120) // Reasonable range: 0-2 hours + { + Logger.LogInformation("Successfully calculated minutes ago from timestamp: {Minutes}", minutesAgo); + return minutesAgo; + } + else + { + Logger.LogWarning("Calculated minutes ago ({Minutes}) outside reasonable range (0-120)", minutesAgo); + } + } + else + { + Logger.LogWarning("Failed to parse timestamp string: '{Timestamp}'", timestampStr); + } + } + else + { + Logger.LogWarning("Display label did not match timestamp pattern. Label: '{Label}', Pattern: '{Pattern}'", trimmedLabel, TextPatterns.TimestampPattern); + } + + return null; + } + catch (Exception ex) + { + Logger.LogDebug(ex, "Failed to extract minutes from display label"); + return null; + } + } + + private bool TryParseTimestamp(string timestampStr, out DateTime timestamp) + { + timestamp = DateTime.MinValue; + + try + { + // Try common Australian date formats + // Format: "Wednesday 17 Dec, 11:05 pm" or "17 Dec, 11:05 pm" + var formats = new[] + { + "dddd d MMM, h:mm tt", // Wednesday 17 Dec, 11:05 pm + "d MMM, h:mm tt", // 17 Dec, 11:05 pm + "dddd d MMM, hh:mm tt", // Wednesday 17 Dec, 11:05 pm (with leading zero) + "d MMM, hh:mm tt", // 17 Dec, 11:05 pm (with leading zero) + "dddd dd MMM, h:mm tt", // Wednesday 17 Dec, 11:05 pm (with leading zero day) + "dd MMM, h:mm tt" // 17 Dec, 11:05 pm (with leading zero day) + }; + + var culture = new System.Globalization.CultureInfo("en-AU"); + + foreach (var format in formats) + { + if (DateTime.TryParseExact(timestampStr, format, culture, + System.Globalization.DateTimeStyles.AssumeLocal, out timestamp)) + { + // If year is not specified, assume current year + if (timestamp.Year == 1) + { + timestamp = new DateTime(DateTime.Now.Year, timestamp.Month, timestamp.Day, + timestamp.Hour, timestamp.Minute, timestamp.Second); + } + + // If the parsed time is in the future (likely same day next year), adjust + if (timestamp > DateTime.Now && timestamp < DateTime.Now.AddDays(1)) + { + // Already correct + } + else if (timestamp > DateTime.Now) + { + // Likely parsed as next year, adjust to this year + timestamp = timestamp.AddYears(-1); + } + + return true; + } + } + + return false; + } + catch + { + return false; + } + } + + private async Task WaitForDisplayLabelToChangeAsync(IPage page, int currentMinutesAgo, int maxWaitMs = 5000) + { + try + { + var startTime = DateTime.UtcNow; + while ((DateTime.UtcNow - startTime).TotalMilliseconds < maxWaitMs) + { + var newMinutesAgo = await ExtractMinutesAgoFromDisplayAsync(page); + if (newMinutesAgo.HasValue && newMinutesAgo.Value != currentMinutesAgo) + { + return; + } + await page.WaitForTimeoutAsync(200); + } + Logger.LogDebug("Display label did not change from {CurrentMinutesAgo} within {MaxWaitMs}ms", currentMinutesAgo, maxWaitMs); + } + catch (Exception ex) + { + Logger.LogDebug(ex, "Error waiting for display label to change"); + } + } + + private async Task DismissModalOverlaysAsync(IPage page) + { + try + { + var hasModal = await page.EvaluateAsync(JavaScriptTemplates.CheckModalOverlay); + + if (!hasModal) + { + return; + } + + Logger.LogDebug("Modal overlay detected, dismissing"); + + await page.Keyboard.PressAsync("Escape"); + await page.WaitForTimeoutAsync(200); + + var stillVisible = await page.EvaluateAsync(JavaScriptTemplates.CheckModalStillVisible); + + if (stillVisible) + { + try + { + var mapContainer = SelectorService.GetLocator(page, Selectors.MapContainer); + await mapContainer.ClickAsync(new LocatorClickOptions { Force = true }); + await page.WaitForTimeoutAsync(100); + } + catch + { + // Ignore if click fails + } + } + } + catch (Exception ex) + { + Logger.LogDebug(ex, "Error dismissing modal overlay, continuing"); + } + } + + private async Task CaptureMapScreenshotAsync(IPage page, ILocator mapContainer, string outputPath, Clip containerClip) + { + if (containerClip == null || containerClip.Width <= 0 || containerClip.Height <= 0) + { + Logger.LogError("Invalid container bounds: X={X}, Y={Y}, Width={Width}, Height={Height}", + containerClip?.X ?? 0, containerClip?.Y ?? 0, containerClip?.Width ?? 0, containerClip?.Height ?? 0); + throw new Exception($"Invalid container bounds: {containerClip?.Width ?? 0}x{containerClip?.Height ?? 0}"); + } + + Clip cropArea; + try + { + cropArea = CalculateCropArea(containerClip); + } + catch (Exception ex) + { + Logger.LogWarning(ex, "Failed to calculate crop area, using full container bounds"); + cropArea = containerClip; + } + + var viewportSize = page.ViewportSize; + int? viewportWidth = viewportSize?.Width; + int? viewportHeight = viewportSize?.Height; + + if (viewportWidth == null || viewportHeight == null) + { + try + { + var viewportJson = await page.EvaluateAsync(JavaScriptTemplates.GetViewportSize); + if (!string.IsNullOrEmpty(viewportJson)) + { + using var doc = System.Text.Json.JsonDocument.Parse(viewportJson); + var root = doc.RootElement; + if (root.TryGetProperty("width", out var widthProp) && root.TryGetProperty("height", out var heightProp)) + { + if (widthProp.TryGetInt32(out var width) && heightProp.TryGetInt32(out var height)) + { + viewportWidth = width; + viewportHeight = height; + } + } + } + } + catch (Exception ex) + { + Logger.LogWarning(ex, "Failed to get viewport size from page evaluation"); + } + + if (viewportWidth == null || viewportHeight == null) + { + viewportWidth = (int)containerClip.Width; + viewportHeight = (int)containerClip.Height; + } + } + + if (viewportWidth.HasValue && viewportHeight.HasValue) + { + if (cropArea.X < 0) + { + cropArea = new Clip { X = 0, Y = cropArea.Y, Width = cropArea.Width + cropArea.X, Height = cropArea.Height }; + } + if (cropArea.Y < 0) + { + cropArea = new Clip { X = cropArea.X, Y = 0, Width = cropArea.Width, Height = cropArea.Height + cropArea.Y }; + } + + if (cropArea.X + cropArea.Width > viewportWidth.Value) + { + var newWidth = viewportWidth.Value - cropArea.X; + cropArea = new Clip { X = cropArea.X, Y = cropArea.Y, Width = newWidth, Height = cropArea.Height }; + } + if (cropArea.Y + cropArea.Height > viewportHeight.Value) + { + var newHeight = viewportHeight.Value - cropArea.Y; + cropArea = new Clip { X = cropArea.X, Y = cropArea.Y, Width = cropArea.Width, Height = newHeight }; + } + } + + if (cropArea.Width <= 0 || cropArea.Height <= 0) + { + Logger.LogError("Invalid crop dimensions after validation: {Width}x{Height}, using full container", cropArea.Width, cropArea.Height); + cropArea = containerClip; + } + + if (cropArea.Width <= 0 || cropArea.Height <= 0) + { + throw new Exception($"Cannot create valid crop area. Container: {containerClip.Width}x{containerClip.Height}, Viewport: {viewportWidth ?? 0}x{viewportHeight ?? 0}"); + } + + try + { + await page.WaitForLoadStateAsync(LoadState.NetworkIdle, new PageWaitForLoadStateOptions { Timeout = 5000 }); + } + catch + { + // Continue if network idle timeout + } + + await DismissModalOverlaysAsync(page); + + await page.ScreenshotAsync(new PageScreenshotOptions + { + Path = outputPath, + Clip = cropArea, + Type = ScreenshotType.Png, + Animations = ScreenshotAnimations.Disabled + }); + + Logger.LogDebug("Screenshot saved: {Path} (crop: {X},{Y} {Width}x{Height})", + outputPath, cropArea.X, cropArea.Y, cropArea.Width, cropArea.Height); + } + + private Clip CalculateCropArea(Clip containerClip) + { + var x = containerClip.X + _cropConfig.X; + var y = containerClip.Y + _cropConfig.Y; + var width = Math.Max(0, containerClip.Width - _cropConfig.X - _cropConfig.RightOffset); + var height = _cropConfig.Height ?? Math.Max(0, containerClip.Height - _cropConfig.Y); + + if (x < containerClip.X || y < containerClip.Y) + { + x = containerClip.X; + y = containerClip.Y; + } + + var maxWidth = containerClip.Width - (x - containerClip.X); + var maxHeight = containerClip.Height - (y - containerClip.Y); + + if (width > maxWidth) + { + width = maxWidth; + } + + if (height > maxHeight) + { + height = maxHeight; + } + + if (width <= 0 || height <= 0) + { + throw new Exception($"Invalid crop dimensions: {width}x{height}"); + } + + return new Clip + { + X = x, + Y = y, + Width = width, + Height = height + }; + } +} + diff --git a/Services/Scraping/Steps/Map/CalculateMapBoundsStep.cs b/Services/Scraping/Steps/Map/CalculateMapBoundsStep.cs new file mode 100644 index 0000000..df8932b --- /dev/null +++ b/Services/Scraping/Steps/Map/CalculateMapBoundsStep.cs @@ -0,0 +1,69 @@ +using BomLocalService.Services.Interfaces; +using BomLocalService.Services.Scraping; +using Microsoft.Playwright; + +namespace BomLocalService.Services.Scraping.Steps.Map; + +public class CalculateMapBoundsStep : BaseScrapingStep +{ + public override string Name => "CalculateMapBounds"; + public override string[] Prerequisites => new[] { "ResetToFirstFrame" }; + + public CalculateMapBoundsStep( + ILogger logger, + ISelectorService selectorService, + IDebugService debugService, + IConfiguration configuration) + : base(logger, selectorService, debugService, configuration) + { + } + + public override bool CanExecute(ScrapingContext context) + { + return context.IsMapReady; + } + + public override async Task ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken) + { + try + { + Logger.LogInformation("Step {Step}: Preparing map container for screenshot", Name); + + var mapContainer = SelectorService.GetLocator(context.Page, Selectors.MapContainer); + await mapContainer.WaitForAsync(new LocatorWaitForOptions { Timeout = 10000 }); + + await context.Page.WaitForFunctionAsync( + JavaScriptTemplates.WaitForMapContainer, + new PageWaitForFunctionOptions { Timeout = 10000 }); + + var boundingBox = await mapContainer.BoundingBoxAsync(); + if (boundingBox == null || boundingBox.Width <= 0 || boundingBox.Height <= 0) + { + var errorMsg = $"Map container has invalid bounds: {boundingBox?.Width ?? 0}x{boundingBox?.Height ?? 0}"; + Logger.LogError("Step {Step}: {Error}", Name, errorMsg); + await SaveErrorDebugAsync(context, errorMsg, cancellationToken); + return ScrapingStepResult.Failed(errorMsg); + } + + var containerClip = new Clip + { + X = boundingBox.X, + Y = boundingBox.Y, + Width = boundingBox.Width, + Height = boundingBox.Height + }; + + context.MapContainer = mapContainer; + context.MapBoundingBox = containerClip; + + return ScrapingStepResult.Successful(); + } + catch (Exception ex) + { + Logger.LogError(ex, "Step {Step} failed", Name); + await SaveErrorDebugAsync(context, $"Failed to calculate map bounds: {ex.Message}", cancellationToken); + return ScrapingStepResult.Failed($"Failed to calculate map bounds: {ex.Message}"); + } + } +} + diff --git a/Services/Scraping/Steps/Map/PauseRadarStep.cs b/Services/Scraping/Steps/Map/PauseRadarStep.cs new file mode 100644 index 0000000..fb0d681 --- /dev/null +++ b/Services/Scraping/Steps/Map/PauseRadarStep.cs @@ -0,0 +1,66 @@ +using BomLocalService.Services.Interfaces; +using BomLocalService.Services.Scraping; +using Microsoft.Playwright; + +namespace BomLocalService.Services.Scraping.Steps.Map; + +public class PauseRadarStep : BaseScrapingStep +{ + public override string Name => "PauseRadar"; + public override string[] Prerequisites => new[] { "WaitForMapReady" }; + + public PauseRadarStep( + ILogger logger, + ISelectorService selectorService, + IDebugService debugService, + IConfiguration configuration) + : base(logger, selectorService, debugService, configuration) + { + } + + public override bool CanExecute(ScrapingContext context) + { + return context.IsMapReady; + } + + public override async Task ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken) + { + try + { + Logger.LogInformation("Step {Step}: Checking if radar loop is paused", Name); + + var playPauseButton = SelectorService.GetLocator(context.Page, Selectors.PlayPauseButton); + await playPauseButton.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000 }); + + var buttonLabel = await playPauseButton.Locator(Selectors.PlayPauseLabel.Selectors[0]).TextContentAsync(); + if (buttonLabel?.Trim().Equals(TextPatterns.ExpectedTexts["PauseButtonLabel"], StringComparison.OrdinalIgnoreCase) == true) + { + Logger.LogInformation("Step {Step}: Radar is playing, pausing it", Name); + await playPauseButton.ClickAsync(); + await context.Page.WaitForTimeoutAsync(300); + + buttonLabel = await playPauseButton.Locator(Selectors.PlayPauseLabel.Selectors[0]).TextContentAsync(); + if (buttonLabel?.Trim().Equals(TextPatterns.ExpectedTexts["PlayButtonLabel"], StringComparison.OrdinalIgnoreCase) != true) + { + Logger.LogWarning("Step {Step}: Radar may not be paused after click, continuing anyway", Name); + } + } + else + { + Logger.LogInformation("Step {Step}: Radar is already paused", Name); + } + + await SaveDebugAsync(context, 8, "radar_paused", cancellationToken); + + context.CurrentState = PageState.SlideshowPaused; + return ScrapingStepResult.Successful(); + } + catch (Exception ex) + { + Logger.LogError(ex, "Step {Step} failed", Name); + await SaveErrorDebugAsync(context, $"Failed to pause radar: {ex.Message}", cancellationToken); + return ScrapingStepResult.Failed($"Failed to pause radar: {ex.Message}"); + } + } +} + diff --git a/Services/Scraping/Steps/Map/ResetToFirstFrameStep.cs b/Services/Scraping/Steps/Map/ResetToFirstFrameStep.cs new file mode 100644 index 0000000..1fa356c --- /dev/null +++ b/Services/Scraping/Steps/Map/ResetToFirstFrameStep.cs @@ -0,0 +1,103 @@ +using BomLocalService.Services.Interfaces; +using BomLocalService.Services.Scraping; +using Microsoft.Playwright; + +namespace BomLocalService.Services.Scraping.Steps.Map; + +public class ResetToFirstFrameStep : BaseScrapingStep +{ + public override string Name => "ResetToFirstFrame"; + public override string[] Prerequisites => new[] { "PauseRadar" }; + + public ResetToFirstFrameStep( + ILogger logger, + ISelectorService selectorService, + IDebugService debugService, + IConfiguration configuration) + : base(logger, selectorService, debugService, configuration) + { + } + + public override bool CanExecute(ScrapingContext context) + { + return context.CurrentState >= PageState.SlideshowPaused; + } + + public override async Task ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken) + { + try + { + Logger.LogInformation("Step {Step}: Resetting to first frame (frame 0)", Name); + + try + { + // Try JavaScript click first to avoid thumb interception + var clicked = await context.Page.EvaluateAsync(@"() => { + const segment = document.querySelector('[data-testid=""bom-scrub-segment""][data-id=""0""]'); + if (segment) { + segment.click(); + return true; + } + return false; + }"); + + if (clicked) + { + await context.Page.WaitForTimeoutAsync(500); + Logger.LogInformation("Step {Step}: Successfully clicked frame 0 segment via JavaScript", Name); + } + else + { + // Fallback to locator click with force + var firstFrameSegment = context.Page.Locator("[data-testid='bom-scrub-segment'][data-id='0']").First; + await firstFrameSegment.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000 }); + await firstFrameSegment.ClickAsync(new LocatorClickOptions { Force = true }); + await context.Page.WaitForTimeoutAsync(500); + Logger.LogInformation("Step {Step}: Successfully clicked frame 0 segment via force click", Name); + } + } + catch (Exception ex) + { + Logger.LogWarning(ex, "Step {Step}: Failed to click first frame segment, continuing anyway", Name); + } + + await SaveDebugAsync(context, 9, "frame_0_selected", cancellationToken); + + // Verify scrubber is at position 0 + Logger.LogInformation("Step {Step}: Verifying scrubber is at position 0", Name); + try + { + var activeSegment = await context.Page.EvaluateAsync(JavaScriptTemplates.CheckActiveFrameSegment); + + if (activeSegment) + { + Logger.LogInformation("Step {Step}: Scrubber confirmed at position 0", Name); + } + else + { + Logger.LogDebug("Step {Step}: Could not confirm scrubber position, but continuing", Name); + } + } + catch (Exception ex) + { + Logger.LogDebug(ex, "Step {Step}: Scrubber position verification failed, continuing anyway", Name); + } + + await SaveDebugAsync(context, 10, "scrubber_at_position_0", cancellationToken); + + // Wait for frame 0 tiles to fully render + var tileRenderWaitMs = Configuration.GetValue("Screenshot:TileRenderWaitMs", 5000); + await context.Page.WaitForTimeoutAsync(tileRenderWaitMs); + + context.CurrentState = PageState.Frame0Selected; + return ScrapingStepResult.Successful(); + } + catch (Exception ex) + { + Logger.LogError(ex, "Step {Step} failed", Name); + await SaveErrorDebugAsync(context, $"Failed to reset to first frame: {ex.Message}", cancellationToken); + return ScrapingStepResult.Failed($"Failed to reset to first frame: {ex.Message}"); + } + } +} + diff --git a/Services/Scraping/Steps/Map/WaitForMapReadyStep.cs b/Services/Scraping/Steps/Map/WaitForMapReadyStep.cs new file mode 100644 index 0000000..d83a73d --- /dev/null +++ b/Services/Scraping/Steps/Map/WaitForMapReadyStep.cs @@ -0,0 +1,73 @@ +using BomLocalService.Services.Interfaces; +using BomLocalService.Services.Scraping; +using Microsoft.Playwright; + +namespace BomLocalService.Services.Scraping.Steps.Map; + +public class WaitForMapReadyStep : BaseScrapingStep +{ + private readonly int _tileRenderWaitMs; + + public override string Name => "WaitForMapReady"; + public override string[] Prerequisites => new[] { "ClickRadarLink" }; + + public WaitForMapReadyStep( + ILogger logger, + ISelectorService selectorService, + IDebugService debugService, + IConfiguration configuration) + : base(logger, selectorService, debugService, configuration) + { + _tileRenderWaitMs = configuration.GetValue("Screenshot:TileRenderWaitMs", 5000); + } + + public override bool CanExecute(ScrapingContext context) + { + return context.IsRadarPageLoaded; + } + + public override async Task ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken) + { + try + { + Logger.LogInformation("Step {Step}: Waiting for weather map page to load", Name); + + await context.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15000 }); + + Logger.LogInformation("Step {Step}: Waiting for map canvas element to render", Name); + var mapCanvas = SelectorService.GetLocator(context.Page, Selectors.MapCanvas); + await mapCanvas.WaitForAsync(new LocatorWaitForOptions { Timeout = 15000 }); + + await context.Page.WaitForFunctionAsync( + JavaScriptTemplates.WaitForMapCanvas, + new PageWaitForFunctionOptions { Timeout = 15000 }); + + Logger.LogInformation("Step {Step}: Map canvas is ready - waiting for map to render", Name); + + try + { + await context.Page.WaitForFunctionAsync( + JavaScriptTemplates.WaitForEsriView, + new PageWaitForFunctionOptions { Timeout = 30000 }); + Logger.LogInformation("Step {Step}: Esri map view is ready", Name); + } + catch + { + Logger.LogInformation("Step {Step}: Esri view ready check timed out, continuing with fixed wait", Name); + } + + await context.Page.WaitForTimeoutAsync(_tileRenderWaitMs); + await SaveDebugAsync(context, 7, "weather_map_ready", cancellationToken); + + context.CurrentState = PageState.MapReady; + return ScrapingStepResult.Successful(); + } + catch (Exception ex) + { + Logger.LogError(ex, "Step {Step} failed", Name); + await SaveErrorDebugAsync(context, $"Failed to wait for map ready: {ex.Message}", cancellationToken); + return ScrapingStepResult.Failed($"Failed to wait for map ready: {ex.Message}"); + } + } +} + diff --git a/Services/Scraping/Steps/Metadata/ExtractMetadataStep.cs b/Services/Scraping/Steps/Metadata/ExtractMetadataStep.cs new file mode 100644 index 0000000..be1076e --- /dev/null +++ b/Services/Scraping/Steps/Metadata/ExtractMetadataStep.cs @@ -0,0 +1,68 @@ +using BomLocalService.Services.Interfaces; +using BomLocalService.Services.Scraping; + +namespace BomLocalService.Services.Scraping.Steps.Metadata; + +public class ExtractMetadataStep : BaseScrapingStep +{ + private readonly ITimeParsingService _timeParsingService; + + public override string Name => "ExtractMetadata"; + public override string[] Prerequisites => new[] { "ResetToFirstFrame" }; + + public ExtractMetadataStep( + ILogger logger, + ISelectorService selectorService, + IDebugService debugService, + IConfiguration configuration, + ITimeParsingService timeParsingService) + : base(logger, selectorService, debugService, configuration) + { + _timeParsingService = timeParsingService; + } + + public override bool CanExecute(ScrapingContext context) + { + return context.IsMapReady; + } + + public override async Task ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken) + { + try + { + Logger.LogInformation("Step {Step}: Extracting metadata and frame information", Name); + + var lastUpdatedInfo = await _timeParsingService.ExtractLastUpdatedInfoAsync(context.Page); + context.LastUpdatedInfo = lastUpdatedInfo; + + // Extract frame info + try + { + var frameInfo = await context.Page.EvaluateAsync(JavaScriptTemplates.ExtractFrameInfo); + var result = new List<(int index, int minutesAgo)>(); + for (int i = 0; i < 7; i++) + { + var minutesAgo = 40 - (i * 5); + result.Add((i, minutesAgo)); + } + context.FrameInfo = result; + } + catch (Exception ex) + { + Logger.LogWarning(ex, "Step {Step}: Failed to extract frame info, using defaults", Name); + context.FrameInfo = Enumerable.Range(0, 7) + .Select(i => (i, 40 - (i * 5))) + .ToList(); + } + + return ScrapingStepResult.Successful(); + } + catch (Exception ex) + { + Logger.LogError(ex, "Step {Step} failed", Name); + await SaveErrorDebugAsync(context, $"Failed to extract metadata: {ex.Message}", cancellationToken); + return ScrapingStepResult.Failed($"Failed to extract metadata: {ex.Message}"); + } + } +} + diff --git a/Services/Scraping/Steps/Navigation/ClickRadarLinkStep.cs b/Services/Scraping/Steps/Navigation/ClickRadarLinkStep.cs new file mode 100644 index 0000000..418d97e --- /dev/null +++ b/Services/Scraping/Steps/Navigation/ClickRadarLinkStep.cs @@ -0,0 +1,56 @@ +using BomLocalService.Services.Interfaces; +using BomLocalService.Services.Scraping; +using Microsoft.Playwright; + +namespace BomLocalService.Services.Scraping.Steps.Navigation; + +public class ClickRadarLinkStep : BaseScrapingStep +{ + public override string Name => "ClickRadarLink"; + public override string[] Prerequisites => new[] { "SelectSearchResult" }; + + public ClickRadarLinkStep( + ILogger logger, + ISelectorService selectorService, + IDebugService debugService, + IConfiguration configuration) + : base(logger, selectorService, debugService, configuration) + { + } + + public override bool CanExecute(ScrapingContext context) + { + return context.IsForecastPageLoaded; + } + + public override async Task ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken) + { + try + { + Logger.LogInformation("Step {Step}: Looking for 'Rain radar and weather map' link", Name); + + var radarLink = await SelectorService.FindElementAsync(context.Page, Selectors.RadarLink, cancellationToken); + + if (radarLink == null) + { + var errorMsg = Selectors.RadarLink.ErrorMessage ?? $"Could not find 'Rain radar and weather map' link for {context.Suburb}, {context.State}"; + await SaveErrorDebugAsync(context, errorMsg, cancellationToken); + return ScrapingStepResult.Failed(errorMsg); + } + + context.RadarLink = radarLink; + await radarLink.ClickAsync(); + await SaveDebugAsync(context, 6, "radar_link_clicked", cancellationToken); + + context.CurrentState = PageState.RadarPageLoaded; + return ScrapingStepResult.Successful(); + } + catch (Exception ex) + { + Logger.LogError(ex, "Step {Step} failed", Name); + await SaveErrorDebugAsync(context, $"Failed to click radar link: {ex.Message}", cancellationToken); + return ScrapingStepResult.Failed($"Failed to click radar link: {ex.Message}"); + } + } +} + diff --git a/Services/Scraping/Steps/Navigation/ClickSearchButtonStep.cs b/Services/Scraping/Steps/Navigation/ClickSearchButtonStep.cs new file mode 100644 index 0000000..7d4c616 --- /dev/null +++ b/Services/Scraping/Steps/Navigation/ClickSearchButtonStep.cs @@ -0,0 +1,65 @@ +using BomLocalService.Services.Interfaces; +using BomLocalService.Services.Scraping; +using Microsoft.Playwright; + +namespace BomLocalService.Services.Scraping.Steps.Navigation; + +public class ClickSearchButtonStep : BaseScrapingStep +{ + public override string Name => "ClickSearchButton"; + public override string[] Prerequisites => new[] { "NavigateHomepage" }; + + public ClickSearchButtonStep( + ILogger logger, + ISelectorService selectorService, + IDebugService debugService, + IConfiguration configuration) + : base(logger, selectorService, debugService, configuration) + { + } + + public override bool CanExecute(ScrapingContext context) + { + return context.IsHomepageLoaded; + } + + public override async Task ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken) + { + try + { + Logger.LogInformation("Step {Step}: Clicking 'Search for a location' button", Name); + + var searchButton = await SelectorService.FindElementAsync(context.Page, Selectors.SearchButton, cancellationToken); + + if (searchButton == null) + { + var errorMsg = Selectors.SearchButton.ErrorMessage ?? "Could not find 'Search for a location' button on BOM homepage."; + await SaveErrorDebugAsync(context, errorMsg, cancellationToken); + return ScrapingStepResult.Failed(errorMsg); + } + + context.SearchButton = searchButton; + await searchButton.ClickAsync(); + + // Wait for search input to appear + var searchInputReady = SelectorService.GetLocator(context.Page, Selectors.SearchInput); + await searchInputReady.WaitForAsync(new LocatorWaitForOptions + { + Timeout = Selectors.SearchInput.TimeoutMs, + State = WaitForSelectorState.Visible + }); + + await SaveDebugAsync(context, 2, "search_button_clicked", cancellationToken); + + context.CurrentState = PageState.SearchModalOpen; + return ScrapingStepResult.Successful(); + } + catch (Exception ex) + { + Logger.LogError(ex, "Step {Step} failed", Name); + await SaveErrorDebugAsync(context, $"Failed to click search button: {ex.Message}", cancellationToken); + return ScrapingStepResult.Failed($"Failed to click search button: {ex.Message}"); + } + } +} + diff --git a/Services/Scraping/Steps/Navigation/NavigateHomepageStep.cs b/Services/Scraping/Steps/Navigation/NavigateHomepageStep.cs new file mode 100644 index 0000000..85a36fa --- /dev/null +++ b/Services/Scraping/Steps/Navigation/NavigateHomepageStep.cs @@ -0,0 +1,62 @@ +using BomLocalService.Services.Interfaces; +using BomLocalService.Services.Scraping; +using Microsoft.Playwright; + +namespace BomLocalService.Services.Scraping.Steps.Navigation; + +public class NavigateHomepageStep : BaseScrapingStep +{ + private readonly string _baseUrl; + + public override string Name => "NavigateHomepage"; + public override string[] Prerequisites => Array.Empty(); + + public NavigateHomepageStep( + ILogger logger, + ISelectorService selectorService, + IDebugService debugService, + IConfiguration configuration) + : base(logger, selectorService, debugService, configuration) + { + _baseUrl = configuration.GetValue("Scraping:BaseUrl") ?? "https://www.bom.gov.au/"; + } + + public override bool CanExecute(ScrapingContext context) + { + return context.CurrentState == PageState.Initial; + } + + public override async Task ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken) + { + try + { + Logger.LogInformation("Step {Step}: Navigating to BOM homepage", Name); + + await context.Page.GotoAsync(_baseUrl, new PageGotoOptions + { + WaitUntil = WaitUntilState.DOMContentLoaded, + Timeout = 30000 + }); + + // Wait for search button to be ready + var searchButtonReady = SelectorService.GetLocator(context.Page, Selectors.SearchButton); + await searchButtonReady.WaitForAsync(new LocatorWaitForOptions + { + Timeout = Selectors.SearchButton.TimeoutMs, + State = WaitForSelectorState.Visible + }); + + await SaveDebugAsync(context, 1, "homepage_loaded", cancellationToken); + + context.CurrentState = PageState.HomepageLoaded; + return ScrapingStepResult.Successful(); + } + catch (Exception ex) + { + Logger.LogError(ex, "Step {Step} failed", Name); + await SaveErrorDebugAsync(context, $"Failed to navigate to homepage: {ex.Message}", cancellationToken); + return ScrapingStepResult.Failed($"Failed to navigate to homepage: {ex.Message}"); + } + } +} + diff --git a/Services/Scraping/Steps/Search/FillSearchInputStep.cs b/Services/Scraping/Steps/Search/FillSearchInputStep.cs new file mode 100644 index 0000000..3a9d6f6 --- /dev/null +++ b/Services/Scraping/Steps/Search/FillSearchInputStep.cs @@ -0,0 +1,48 @@ +using BomLocalService.Services.Interfaces; +using BomLocalService.Services.Scraping; +using Microsoft.Playwright; + +namespace BomLocalService.Services.Scraping.Steps.Search; + +public class FillSearchInputStep : BaseScrapingStep +{ + public override string Name => "FillSearchInput"; + public override string[] Prerequisites => new[] { "ClickSearchButton" }; + + public FillSearchInputStep( + ILogger logger, + ISelectorService selectorService, + IDebugService debugService, + IConfiguration configuration) + : base(logger, selectorService, debugService, configuration) + { + } + + public override bool CanExecute(ScrapingContext context) + { + return context.IsSearchModalOpen; + } + + public override async Task ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken) + { + try + { + Logger.LogInformation("Step {Step}: Searching for suburb: {Suburb}", Name, context.Suburb); + + var searchInput = SelectorService.GetLocator(context.Page, Selectors.SearchInput); + await searchInput.FillAsync(context.Suburb); + + context.SearchInput = searchInput; + await SaveDebugAsync(context, 3, "search_input_filled", cancellationToken); + + return ScrapingStepResult.Successful(); + } + catch (Exception ex) + { + Logger.LogError(ex, "Step {Step} failed", Name); + await SaveErrorDebugAsync(context, $"Failed to fill search input: {ex.Message}", cancellationToken); + return ScrapingStepResult.Failed($"Failed to fill search input: {ex.Message}"); + } + } +} + diff --git a/Services/Scraping/Steps/Search/SelectSearchResultStep.cs b/Services/Scraping/Steps/Search/SelectSearchResultStep.cs new file mode 100644 index 0000000..433d431 --- /dev/null +++ b/Services/Scraping/Steps/Search/SelectSearchResultStep.cs @@ -0,0 +1,192 @@ +using BomLocalService.Services.Interfaces; +using BomLocalService.Services.Scraping; +using BomLocalService.Utilities; +using Microsoft.Playwright; +using System.Text.RegularExpressions; + +namespace BomLocalService.Services.Scraping.Steps.Search; + +public class SelectSearchResultStep : BaseScrapingStep +{ + public override string Name => "SelectSearchResult"; + public override string[] Prerequisites => new[] { "WaitForSearchResults" }; + + public SelectSearchResultStep( + ILogger logger, + ISelectorService selectorService, + IDebugService debugService, + IConfiguration configuration) + : base(logger, selectorService, debugService, configuration) + { + } + + public override bool CanExecute(ScrapingContext context) + { + return context.CurrentState >= PageState.SearchResultsVisible; + } + + public override async Task ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken) + { + try + { + Logger.LogInformation("Step {Step}: Looking for matching search result for {Suburb}, {State}", Name, context.Suburb, context.State); + + var suburbLower = context.Suburb.ToLower().Trim(); + var stateLower = context.State.ToLower().Trim(); + + // Get the actual count from the summary element + var resultsTitle = SelectorService.GetLocator(context.Page, Selectors.ResultsTitle); + var summaryText = await resultsTitle.TextContentAsync(); + int? actualCount = null; + + if (!string.IsNullOrEmpty(summaryText)) + { + var countMatch = Regex.Match(summaryText, TextPatterns.ResultsCountPattern, RegexOptions.IgnoreCase); + if (countMatch.Success && countMatch.Groups.Count >= 3 && countMatch.Groups[2].Success) + { + if (int.TryParse(countMatch.Groups[2].Value, out var total)) + { + actualCount = total; + } + } + } + + // Fetch all result data + List<(string name, string desc, string fullText)> results = new(); + try + { + var resultData = await context.Page.EvaluateAsync(JavaScriptTemplates.ExtractSearchResults); + + results = resultData.Select(arr => ( + name: arr.Length > 0 ? arr[0] : "", + desc: arr.Length > 1 ? arr[1] : "", + fullText: arr.Length > 2 ? arr[2] : "" + )).ToList(); + } + catch (Exception ex) + { + Logger.LogWarning(ex, "Failed to extract structured result data, falling back to text content"); + var resultTexts = await context.Page.EvaluateAsync(JavaScriptTemplates.ExtractSearchResultsFallback); + results = resultTexts.Select(text => (name: "", desc: "", fullText: text)).ToList(); + } + + if (actualCount.HasValue) + { + Logger.LogInformation("Found {Count} location results (summary: {Summary})", actualCount.Value, summaryText?.Trim()); + if (results.Count > actualCount.Value) + { + results = results.Take(actualCount.Value).ToList(); + } + } + else + { + Logger.LogInformation("Found {Count} search results", results.Count); + } + + int? matchingIndex = null; + int bestMatchScore = -1; + + for (int i = 0; i < results.Count; i++) + { + var (name, desc, fullText) = results[i]; + var nameLower = name.ToLower().Trim(); + var descLower = desc.ToLower().Trim(); + var fullTextLower = fullText.ToLower(); + + Logger.LogInformation("Checking result {Index}: Name='{Name}', Desc='{Desc}', FullText='{FullText}'", + i, name, desc, fullText.Length > 100 ? fullText.Substring(0, 100) + "..." : fullText); + + var matchesSuburb = false; + var matchScore = 0; + + if (!string.IsNullOrEmpty(name)) + { + if (nameLower == suburbLower) + { + matchesSuburb = true; + matchScore = 100; + } + else if (nameLower.StartsWith(suburbLower + " ") || nameLower.StartsWith(suburbLower + "(")) + { + matchesSuburb = true; + matchScore = 80; + } + else if (nameLower.Contains("(" + suburbLower + ")") || nameLower.Contains("(" + suburbLower + " ")) + { + matchesSuburb = true; + matchScore = 60; + } + else if (nameLower.Contains(suburbLower)) + { + matchesSuburb = true; + matchScore = 40; + } + } + else + { + if (fullTextLower.Contains(suburbLower)) + { + matchesSuburb = true; + matchScore = 20; + } + } + + var matchesState = false; + if (!string.IsNullOrEmpty(desc)) + { + matchesState = StateAbbreviationHelper.MatchesState(descLower, stateLower); + } + if (!matchesState) + { + matchesState = StateAbbreviationHelper.MatchesState(fullTextLower, stateLower); + } + + Logger.LogInformation("Result {Index}: matchesSuburb={MatchesSuburb} (score={Score}), matchesState={MatchesState}", + i, matchesSuburb, matchScore, matchesState); + + if (matchesSuburb && matchesState && matchScore > bestMatchScore) + { + matchingIndex = i; + bestMatchScore = matchScore; + Logger.LogInformation("New best match found: {Name} - {Desc} (score: {Score})", name, desc, matchScore); + } + } + + if (matchingIndex.HasValue) + { + Logger.LogInformation("Using best matching result at index {Index} with score {Score}", matchingIndex.Value, bestMatchScore); + } + else + { + Logger.LogInformation("No exact match found, using first result"); + } + + context.SearchResults = results; + context.SelectedResultIndex = matchingIndex; + + // Click the matching result + var resultsList = context.Page.Locator("ul[aria-labelledby='location-results-title']"); + var allResults = resultsList.Locator("li.bom-linklist__item[role='listitem']"); + var resultToClick = matchingIndex.HasValue ? allResults.Nth(matchingIndex.Value) : allResults.First; + + await resultToClick.ClickAsync(); + await SaveDebugAsync(context, 4, "search_result_selected", cancellationToken); + + // Wait for forecast page to load + await context.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15000 }); + var dynamicContentWaitMs = Configuration.GetValue("Screenshot:DynamicContentWaitMs", 2000); + await context.Page.WaitForTimeoutAsync(dynamicContentWaitMs); + await SaveDebugAsync(context, 5, "forecast_page_loaded", cancellationToken); + + context.CurrentState = PageState.ForecastPageLoaded; + return ScrapingStepResult.Successful(); + } + catch (Exception ex) + { + Logger.LogError(ex, "Step {Step} failed", Name); + await SaveErrorDebugAsync(context, $"Failed to select search result: {ex.Message}", cancellationToken); + return ScrapingStepResult.Failed($"Failed to select search result: {ex.Message}"); + } + } +} + diff --git a/Services/Scraping/Steps/Search/WaitForSearchResultsStep.cs b/Services/Scraping/Steps/Search/WaitForSearchResultsStep.cs new file mode 100644 index 0000000..c53baa4 --- /dev/null +++ b/Services/Scraping/Steps/Search/WaitForSearchResultsStep.cs @@ -0,0 +1,47 @@ +using BomLocalService.Services.Interfaces; +using BomLocalService.Services.Scraping; +using Microsoft.Playwright; + +namespace BomLocalService.Services.Scraping.Steps.Search; + +public class WaitForSearchResultsStep : BaseScrapingStep +{ + public override string Name => "WaitForSearchResults"; + public override string[] Prerequisites => new[] { "FillSearchInput" }; + + public WaitForSearchResultsStep( + ILogger logger, + ISelectorService selectorService, + IDebugService debugService, + IConfiguration configuration) + : base(logger, selectorService, debugService, configuration) + { + } + + public override bool CanExecute(ScrapingContext context) + { + return context.IsSearchModalOpen; + } + + public override async Task ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken) + { + try + { + Logger.LogInformation("Step {Step}: Waiting for autocomplete suggestions", Name); + + await context.Page.WaitForFunctionAsync( + JavaScriptTemplates.WaitForSearchResults, + new PageWaitForFunctionOptions { Timeout = 10000 }); + + context.CurrentState = PageState.SearchResultsVisible; + return ScrapingStepResult.Successful(); + } + catch (Exception ex) + { + Logger.LogError(ex, "Step {Step} failed", Name); + await SaveErrorDebugAsync(context, $"Failed to wait for search results: {ex.Message}", cancellationToken); + return ScrapingStepResult.Failed($"Failed to wait for search results: {ex.Message}"); + } + } +} + diff --git a/Services/Scraping/WorkflowFactory.cs b/Services/Scraping/WorkflowFactory.cs new file mode 100644 index 0000000..3f18978 --- /dev/null +++ b/Services/Scraping/WorkflowFactory.cs @@ -0,0 +1,22 @@ +namespace BomLocalService.Services.Scraping; + +public class WorkflowFactory : IWorkflowFactory +{ + private readonly IServiceProvider _serviceProvider; + + public WorkflowFactory(IServiceProvider serviceProvider) + { + _serviceProvider = serviceProvider; + } + + public IWorkflow GetWorkflow(string name) + { + return name switch + { + "RadarScraping" => (IWorkflow)_serviceProvider.GetRequiredService(), + "TemperatureMap" => (IWorkflow)_serviceProvider.GetRequiredService(), + _ => throw new ArgumentException($"Unknown workflow: {name}") + }; + } +} + diff --git a/Services/Scraping/Workflows/RadarScrapingWorkflow.cs b/Services/Scraping/Workflows/RadarScrapingWorkflow.cs new file mode 100644 index 0000000..4358746 --- /dev/null +++ b/Services/Scraping/Workflows/RadarScrapingWorkflow.cs @@ -0,0 +1,210 @@ +using BomLocalService.Models; +using BomLocalService.Services.Interfaces; +using BomLocalService.Services.Scraping; +using BomLocalService.Utilities; + +namespace BomLocalService.Services.Scraping.Workflows; + +public class RadarScrapingWorkflow : IWorkflow +{ + private readonly ILogger _logger; + private readonly IScrapingStepRegistry _stepRegistry; + private readonly IConfiguration _configuration; + private readonly ICacheService _cacheService; + private readonly double _cacheExpirationMinutes; + private readonly int _cacheManagementCheckIntervalMinutes; + + public string Name => "RadarScraping"; + public string Description => "Scrapes radar images for a location"; + + // Fixed step sequence - order matters due to dependencies + public string[] StepNames => new[] + { + "NavigateHomepage", + "ClickSearchButton", + "FillSearchInput", + "WaitForSearchResults", + "SelectSearchResult", + "ClickRadarLink", + "WaitForMapReady", + "PauseRadar", + "ResetToFirstFrame", + "ExtractMetadata", + "CalculateMapBounds", + "CaptureFrames" + }; + + public RadarScrapingWorkflow( + ILogger logger, + IScrapingStepRegistry stepRegistry, + IConfiguration configuration, + ICacheService cacheService) + { + _logger = logger; + _stepRegistry = stepRegistry; + _configuration = configuration; + _cacheService = cacheService; + _cacheExpirationMinutes = configuration.GetValue("CacheExpirationMinutes", 12.5); + _cacheManagementCheckIntervalMinutes = configuration.GetValue("CacheManagement:CheckIntervalMinutes", 5); + } + + public async Task ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken) + { + var workflowConfig = _configuration.GetSection($"Scraping:Workflows:{Name}").Get(); + var workflowStartTime = DateTime.UtcNow; + var stepTimings = new List<(string stepName, double durationSeconds)>(); + + // Record Initializing phase start for metrics tracking + if (!string.IsNullOrEmpty(context.CacheFolderPath)) + { + _cacheService.RecordUpdateProgressByFolder(context.CacheFolderPath, CacheUpdatePhase.Initializing); + } + + foreach (var stepName in StepNames) + { + var stepConfig = workflowConfig?.Steps?.GetValueOrDefault(stepName) ?? new ScrapingStepConfig { Enabled = true }; + + if (!stepConfig.Enabled) + { + _logger.LogInformation("Step {Step} is disabled, skipping", stepName); + continue; + } + + var step = _stepRegistry.GetStep(stepName); + if (step == null) + { + throw new InvalidOperationException($"Step {stepName} not found in registry"); + } + + if (!ValidatePrerequisites(step, context)) + { + throw new InvalidOperationException( + $"Step {stepName} prerequisites not met. Required: {string.Join(", ", step.Prerequisites)}"); + } + + if (!step.CanExecute(context)) + { + throw new InvalidOperationException( + $"Step {stepName} cannot execute in current page state: {context.CurrentState}"); + } + + var stepStartTime = DateTime.UtcNow; + _logger.LogInformation("Executing step {Step}", stepName); + + var result = await step.ExecuteAsync(context, cancellationToken); + + var stepDuration = (DateTime.UtcNow - stepStartTime).TotalSeconds; + stepTimings.Add((stepName, stepDuration)); + + // Record step timing in metrics + _cacheService.RecordStepCompletion(stepName, stepDuration); + + // Get historical average for comparison + var avgDuration = _cacheService.GetAverageStepDuration(stepName); + if (avgDuration > 0) + { + var diff = stepDuration - avgDuration; + var diffPercent = (diff / avgDuration) * 100; + var isSlow = stepDuration > avgDuration * 1.5; // 50% slower than average + + if (isSlow) + { + _logger.LogWarning("Step {Step} took significantly longer than average: {Duration:F2}s (avg: {Avg:F2}s, {Diff:+#.##}s, {DiffPercent:+#0.#}% slower)", + stepName, stepDuration, avgDuration, diff, diffPercent); + } + else if (Math.Abs(diffPercent) > 20) // More than 20% difference (faster or slower) + { + _logger.LogInformation("Step {Step} completed in {Duration:F2}s (avg: {Avg:F2}s, {Diff:+#.##;-#.##}s, {DiffPercent:+#0.#;-#0.#}%)", + stepName, stepDuration, avgDuration, diff, diffPercent); + } + else + { + _logger.LogInformation("Step {Step} completed in {Duration:F2}s (avg: {Avg:F2}s)", stepName, stepDuration, avgDuration); + } + } + else + { + _logger.LogInformation("Step {Step} completed in {Duration:F2}s", stepName, stepDuration); + } + + if (!result.Success) + { + throw new Exception($"Step {stepName} failed: {result.ErrorMessage}"); + } + + context.CompletedSteps.Add(stepName); + } + + var totalDuration = (DateTime.UtcNow - workflowStartTime).TotalSeconds; + + // Check if workflow duration is significantly longer than average + var avgTotalDuration = _cacheService.GetAverageTotalDuration(); + var isWorkflowSlow = avgTotalDuration > 0 && totalDuration > avgTotalDuration * 1.3; // 30% slower than average + + // Log step breakdown + var stepBreakdown = string.Join(", ", stepTimings.Select(t => $"{t.stepName}={t.durationSeconds:F2}s")); + + if (isWorkflowSlow) + { + var workflowDiff = totalDuration - avgTotalDuration; + var workflowDiffPercent = (workflowDiff / avgTotalDuration) * 100; + _logger.LogWarning("Workflow {Workflow} took significantly longer than average: {TotalDuration:F2}s (avg: {Avg:F2}s, {Diff:+#.##}s, {DiffPercent:+#0.#}% slower) for {Suburb}, {State}", + Name, totalDuration, avgTotalDuration, workflowDiff, workflowDiffPercent, context.Suburb, context.State); + } + else + { + _logger.LogInformation("Workflow {Workflow} completed in {TotalDuration:F2}s for {Suburb}, {State}. Step breakdown: {StepBreakdown}", + Name, totalDuration, context.Suburb, context.State, stepBreakdown); + } + + // Log step performance summary if we have historical data + var stepMetrics = _cacheService.GetStepMetrics(); + if (stepMetrics.Count > 0) + { + var slowSteps = stepTimings + .Where(t => stepMetrics.ContainsKey(t.stepName) && t.durationSeconds > stepMetrics[t.stepName] * 1.5) + .Select(t => $"{t.stepName} ({t.durationSeconds:F2}s vs avg {stepMetrics[t.stepName]:F2}s)") + .ToList(); + + if (slowSteps.Any()) + { + _logger.LogWarning("Workflow {Workflow} had slower-than-average steps: {SlowSteps}", + Name, string.Join(", ", slowSteps)); + } + } + + return BuildResponse(context); + } + + private bool ValidatePrerequisites(IScrapingStep step, ScrapingContext context) + { + return step.Prerequisites.All(prereq => context.CompletedSteps.Contains(prereq)); + } + + private RadarResponse BuildResponse(ScrapingContext context) + { + if (context.LastUpdatedInfo == null) + { + throw new InvalidOperationException("LastUpdatedInfo is required to build response"); + } + + if (context.Frames == null || context.Frames.Count == 0) + { + throw new InvalidOperationException("Frames are required to build response"); + } + + var cacheExpiresAt = context.LastUpdatedInfo.ObservationTime.AddMinutes(_cacheExpirationMinutes); + + return ResponseBuilder.CreateRadarResponse( + context.CacheFolderPath, + context.Frames, + _cacheManagementCheckIntervalMinutes, + context.LastUpdatedInfo, + context.Suburb, + context.State, + cacheIsValid: true, + cacheExpiresAt: cacheExpiresAt, + isUpdating: false); + } +} + diff --git a/Services/Scraping/Workflows/TemperatureMapWorkflow.cs b/Services/Scraping/Workflows/TemperatureMapWorkflow.cs new file mode 100644 index 0000000..4176105 --- /dev/null +++ b/Services/Scraping/Workflows/TemperatureMapWorkflow.cs @@ -0,0 +1,41 @@ +using BomLocalService.Models; +using BomLocalService.Services.Interfaces; +using BomLocalService.Services.Scraping; + +namespace BomLocalService.Services.Scraping.Workflows; + +/// +/// Future workflow for temperature map scraping (not yet implemented) +/// +public class TemperatureMapWorkflow : IWorkflow +{ + private readonly ILogger _logger; + + public string Name => "TemperatureMap"; + public string Description => "Scrapes temperature forecast map (future feature)"; + + public string[] StepNames => new[] + { + "NavigateHomepage", + "ClickSearchButton", + "FillSearchInput", + "WaitForSearchResults", + "SelectSearchResult", + // Future: "ClickTemperatureMapLink", + // Future: "WaitForMapReady", + // Future: "ExtractMetadata", + // Future: "CalculateMapBounds", + // Future: "CaptureTemperatureFrames" + }; + + public TemperatureMapWorkflow(ILogger logger) + { + _logger = logger; + } + + public Task ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken) + { + throw new NotImplementedException("TemperatureMap workflow is not yet implemented"); + } +} + diff --git a/Services/ScrapingService.cs b/Services/ScrapingService.cs index 640e542..67a9609 100644 --- a/Services/ScrapingService.cs +++ b/Services/ScrapingService.cs @@ -1,6 +1,6 @@ using BomLocalService.Models; using BomLocalService.Services.Interfaces; -using BomLocalService.Utilities; +using BomLocalService.Services.Scraping; using Microsoft.Playwright; namespace BomLocalService.Services; @@ -8,83 +8,16 @@ namespace BomLocalService.Services; public class ScrapingService : IScrapingService { private readonly ILogger _logger; - private readonly IBrowserService _browserService; - private readonly ITimeParsingService _timeParsingService; - private readonly ICacheService _cacheService; - private readonly IDebugService _debugService; - private readonly IConfiguration _configuration; - private readonly int _dynamicContentWaitMs; - private readonly int _tileRenderWaitMs; - private readonly ScreenshotCropConfig _cropConfig; - private readonly double _cacheExpirationMinutes; - private readonly int _cacheManagementCheckIntervalMinutes; - - // Selector constants - private static readonly string[] SearchButtonSelectors = new[] - { - "button[data-testid='searchLabel']", - "button[aria-label='Search for a location']", - "button.search-location__trigger-button", - "button.bom-button:has(span.bom-button__label:has-text('Search for a location'))", - "button:has-text('Search for a location')", - "[data-testid='showSearchModal'] button" - }; - - private static readonly string[] RadarLinkSelectors = new[] - { - "a.jump-link.bom-button.bom-button--secondary:has-text('Rain radar and weather map')", - ".cta.button a.jump-link:has-text('Rain radar and weather map')", - "a.jump-link:has-text('Rain radar and weather map')", - ".cta.button a:has-text('Rain radar and weather map')", - "a.bom-button--secondary:has-text('Rain radar and weather map')", - "a:has-text('Rain radar and weather map')", - "a:has-text('rain radar')", - "a[href*='radar']" - }; - + private readonly IWorkflowFactory _workflowFactory; + public ScrapingService( ILogger logger, - IBrowserService browserService, - ITimeParsingService timeParsingService, - ICacheService cacheService, - IDebugService debugService, - IConfiguration configuration) + IWorkflowFactory workflowFactory) { _logger = logger; - _browserService = browserService; - _timeParsingService = timeParsingService; - _cacheService = cacheService; - _debugService = debugService; - _configuration = configuration; - _dynamicContentWaitMs = configuration.GetValue("Screenshot:DynamicContentWaitMs", 2000); - _tileRenderWaitMs = configuration.GetValue("Screenshot:TileRenderWaitMs", 5000); - - // Load crop configuration - var cropSection = configuration.GetSection("Screenshot:Crop"); - _cropConfig = new ScreenshotCropConfig - { - X = cropSection.GetValue("X", 0), - Y = cropSection.GetValue("Y", 0), - RightOffset = cropSection.GetValue("RightOffset", 0), - Height = cropSection.GetValue("Height") - }; - - _logger.LogInformation("Screenshot crop config: X={X}, Y={Y}, RightOffset={RightOffset}, Height={Height}", - _cropConfig.X, _cropConfig.Y, _cropConfig.RightOffset, _cropConfig.Height); - - _cacheExpirationMinutes = configuration.GetValue("CacheExpirationMinutes", 12.5); - _cacheManagementCheckIntervalMinutes = configuration.GetValue("CacheManagement:CheckIntervalMinutes", 5); - - if (_cacheExpirationMinutes <= 0) - { - throw new ArgumentException("CacheExpirationMinutes must be greater than 0", nameof(configuration)); - } - if (_cacheManagementCheckIntervalMinutes <= 0 || _cacheManagementCheckIntervalMinutes > 60) - { - throw new ArgumentException("CacheManagement:CheckIntervalMinutes must be between 1 and 60", nameof(configuration)); - } + _workflowFactory = workflowFactory; } - + /// /// Scrapes the BOM website to get a radar screenshot for a location /// @@ -98,941 +31,22 @@ public class ScrapingService : IScrapingService List<(string method, string url, int? status, string resourceType, DateTime timestamp)> networkRequests, CancellationToken cancellationToken = default) { - try + var context = new ScrapingContext { - // Step 1: Navigate to BOM homepage and wait for search button - _logger.LogInformation("Navigating to BOM homepage"); - await page.GotoAsync("https://www.bom.gov.au/", new PageGotoOptions - { - WaitUntil = WaitUntilState.DOMContentLoaded, - Timeout = 30000 - }); - - // Wait for search button to be ready instead of NetworkIdle (faster) - var searchButtonReady = page.Locator("button[data-testid='searchLabel'], button[aria-label='Search for a location'], button.search-location__trigger-button").First; - await searchButtonReady.WaitForAsync(new LocatorWaitForOptions { Timeout = 10000, State = WaitForSelectorState.Visible }); - await _debugService.SaveStepDebugAsync(debugFolder, 1, "homepage_loaded", page, consoleMessages, networkRequests, cancellationToken); - - // Step 2: Click "Search for a location" button to open search UI - _logger.LogInformation("Clicking 'Search for a location' button"); - var searchButton = await SelectorHelper.FindLocatorBySelectorsAsync(page, SearchButtonSelectors, _logger); - - if (searchButton == null) - { - var errorMsg = "Could not find 'Search for a location' button on BOM homepage."; - await _debugService.SaveErrorDebugAsync(debugFolder, errorMsg, page, consoleMessages, networkRequests, cancellationToken); - throw new Exception(errorMsg); - } - - await searchButton.ClickAsync(); - - // Wait for search input to appear - var searchInputReady = page.Locator("#search-enter-keyword").First; - await searchInputReady.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000, State = WaitForSelectorState.Visible }); - await _debugService.SaveStepDebugAsync(debugFolder, 2, "search_button_clicked", page, consoleMessages, networkRequests, cancellationToken); - - // Step 3: Find and fill search box with suburb name - _logger.LogInformation("Searching for suburb: {Suburb}", suburb); - var searchInput = page.Locator("#search-enter-keyword").First; - await searchInput.FillAsync(suburb); - - // Step 4: Wait for autocomplete suggestions to appear - _logger.LogInformation("Waiting for autocomplete suggestions"); - await page.WaitForFunctionAsync(@"() => { - const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=""listitem""]')); - return results.length > 0 && results.some(r => r.offsetParent !== null); - }", new PageWaitForFunctionOptions { Timeout = 10000 }); - await _debugService.SaveStepDebugAsync(debugFolder, 3, "search_input_filled", page, consoleMessages, networkRequests, cancellationToken); - - // Step 5: Find the matching search result based on suburb and state - _logger.LogInformation("Looking for matching search result for {Suburb}, {State}", suburb, state); - var suburbLower = suburb.ToLower().Trim(); - var stateLower = state.ToLower().Trim(); - - // Get the actual count from the summary element - var summaryText = await page.Locator("#location-results-title, [data-testid='location-results-title']").First.TextContentAsync(); - int? actualCount = null; - if (!string.IsNullOrEmpty(summaryText)) - { - // Parse "3 of 3 location results" or similar - // Pattern has 2 capture groups: (\d+) of (\d+) - // Groups[0] = full match, Groups[1] = first number, Groups[2] = second number (total) - var countMatch = System.Text.RegularExpressions.Regex.Match(summaryText, @"(\d+)\s+of\s+(\d+)", System.Text.RegularExpressions.RegexOptions.IgnoreCase); - if (countMatch.Success && countMatch.Groups.Count >= 3 && countMatch.Groups[2].Success) - { - if (int.TryParse(countMatch.Groups[2].Value, out var total)) - { - actualCount = total; - } - } - } - - // Fetch all result data in one JavaScript evaluation - extract location name and state separately - List<(string name, string desc, string fullText)> results = new(); - try - { - var resultData = await page.EvaluateAsync(@"() => { - // Scope to the location results list - find the ul with aria-labelledby pointing to location-results-title - const resultsList = document.querySelector('ul[aria-labelledby=""location-results-title""]'); - if (!resultsList) { - console.log('Location results list not found'); - return []; - } - const results = Array.from(resultsList.querySelectorAll('li.bom-linklist__item[role=""listitem""]')); - console.log('Found', results.length, 'location results'); - return results.map((r, index) => { - // Query from the li element - elements are nested inside tag - const nameEl = r.querySelector('[data-testid=""location-name""]'); - const descEl = r.querySelector('.bom-linklist-item__desc'); - const name = nameEl ? (nameEl.textContent || nameEl.innerText || '').trim() : ''; - const desc = descEl ? (descEl.textContent || descEl.innerText || '').trim() : ''; - const fullText = (r.textContent || r.innerText || '').trim(); - console.log('Result', index, ':', { hasNameEl: !!nameEl, hasDescEl: !!descEl, name: name, desc: desc }); - return [name, desc, fullText]; - }); - }"); - - // Convert to structured data - results = resultData.Select(arr => ( - name: arr.Length > 0 ? arr[0] : "", - desc: arr.Length > 1 ? arr[1] : "", - fullText: arr.Length > 2 ? arr[2] : "" - )).ToList(); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to extract structured result data, falling back to text content"); - // Fallback to simple text extraction - var resultTexts = await page.EvaluateAsync(@"() => { - const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=""listitem""]')); - return results.map(r => r.textContent || ''); - }"); - results = resultTexts.Select(text => (name: "", desc: "", fullText: text)).ToList(); - } - - if (actualCount.HasValue) - { - _logger.LogInformation("Found {Count} location results (summary: {Summary})", actualCount.Value, summaryText?.Trim()); - // Only use the first N results that match the actual count - if (results.Count > actualCount.Value) - { - results = results.Take(actualCount.Value).ToList(); - } - } - else - { - _logger.LogInformation("Found {Count} search results", results.Count); - } - - int? matchingIndex = null; - int bestMatchScore = -1; - - for (int i = 0; i < results.Count; i++) - { - var (name, desc, fullText) = results[i]; - var nameLower = name.ToLower().Trim(); - var descLower = desc.ToLower().Trim(); - var fullTextLower = fullText.ToLower(); - - _logger.LogInformation("Checking result {Index}: Name='{Name}', Desc='{Desc}', FullText='{FullText}'", - i, name, desc, fullText.Length > 100 ? fullText.Substring(0, 100) + "..." : fullText); - - // Check if suburb name matches with priority scoring - var matchesSuburb = false; - var matchScore = 0; - - if (!string.IsNullOrEmpty(name)) - { - if (nameLower == suburbLower) - { - // Exact match - highest priority - matchesSuburb = true; - matchScore = 100; - } - else if (nameLower.StartsWith(suburbLower + " ") || nameLower.StartsWith(suburbLower + "(")) - { - // Starts with suburb name (e.g., "Sydney" or "Sydney (Inner)") - matchesSuburb = true; - matchScore = 80; - } - else if (nameLower.Contains("(" + suburbLower + ")") || nameLower.Contains("(" + suburbLower + " ")) - { - // Suburb name in parentheses (e.g., "Darlington (Sydney)") - lower priority - matchesSuburb = true; - matchScore = 60; - } - else if (nameLower.Contains(suburbLower)) - { - // Contains suburb name anywhere - lowest priority - matchesSuburb = true; - matchScore = 40; - } - } - else - { - // Fallback to fullText if name extraction failed - if (fullTextLower.Contains(suburbLower)) - { - matchesSuburb = true; - matchScore = 20; - } - } - - // Check if state matches (from description which contains "Queensland 4300" or "New South Wales 2469") - var matchesState = false; - if (!string.IsNullOrEmpty(desc)) - { - matchesState = StateAbbreviationHelper.MatchesState(descLower, stateLower); - } - // Always check fullText for state as fallback - if (!matchesState) - { - matchesState = StateAbbreviationHelper.MatchesState(fullTextLower, stateLower); - } - - _logger.LogInformation("Result {Index}: matchesSuburb={MatchesSuburb} (score={Score}, suburb='{Suburb}' vs name='{Name}'), matchesState={MatchesState} (state='{State}' vs desc='{Desc}')", - i, matchesSuburb, matchScore, suburbLower, nameLower, matchesState, stateLower, descLower); - - // Track the best matching result (highest score) - if (matchesSuburb && matchesState && matchScore > bestMatchScore) - { - matchingIndex = i; - bestMatchScore = matchScore; - _logger.LogInformation("New best match found: {Name} - {Desc} (score: {Score})", name, desc, matchScore); - } - } - - // Use the best match found (or first if no match) - if (matchingIndex.HasValue) - { - _logger.LogInformation("Using best matching result at index {Index} with score {Score}", matchingIndex.Value, bestMatchScore); - } - else - { - _logger.LogInformation("No exact match found, using first result"); - } - - // Click the matching result (or first if no match) - // Scope to the location results list - var resultsList = page.Locator("ul[aria-labelledby='location-results-title']"); - var allResults = resultsList.Locator("li.bom-linklist__item[role='listitem']"); - var resultToClick = matchingIndex.HasValue ? allResults.Nth(matchingIndex.Value) : allResults.First; - - if (!matchingIndex.HasValue) - { - _logger.LogInformation("No exact match found, using first result"); - } - - await resultToClick.ClickAsync(); - await _debugService.SaveStepDebugAsync(debugFolder, 4, "search_result_selected", page, consoleMessages, networkRequests, cancellationToken); - - // Wait for forecast page to load - await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15000 }); - await page.WaitForTimeoutAsync(_dynamicContentWaitMs); // Brief wait for dynamic content - await _debugService.SaveStepDebugAsync(debugFolder, 5, "forecast_page_loaded", page, consoleMessages, networkRequests, cancellationToken); - - // Step 6: Find and click "Rain radar and weather map" link - _logger.LogInformation("Looking for 'Rain radar and weather map' link"); - var radarLink = await SelectorHelper.FindLocatorBySelectorsAsync(page, RadarLinkSelectors, _logger); - - if (radarLink == null) - { - var errorMsg = $"Could not find 'Rain radar and weather map' link for {suburb}, {state}"; - await _debugService.SaveErrorDebugAsync(debugFolder, errorMsg, page, consoleMessages, networkRequests, cancellationToken); - throw new Exception(errorMsg); - } - - await radarLink.ClickAsync(); - await _debugService.SaveStepDebugAsync(debugFolder, 6, "radar_link_clicked", page, consoleMessages, networkRequests, cancellationToken); - - // Step 7: Wait for weather map page to load and map to fully render - _logger.LogInformation("Waiting for weather map page to load"); - await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15000 }); - - // Wait for the map canvas element to appear with proper dimensions - _logger.LogInformation("Waiting for map canvas element to render"); - var mapCanvas = page.Locator(".esri-view-surface canvas").First; - await mapCanvas.WaitForAsync(new LocatorWaitForOptions { Timeout = 15000 }); - - // Wait for canvas to have valid dimensions - await page.WaitForFunctionAsync(@"() => { - const canvas = document.querySelector('.esri-view-surface canvas'); - return canvas && canvas.width > 0 && canvas.height > 0 && canvas.offsetWidth > 0 && canvas.offsetHeight > 0; - }", new PageWaitForFunctionOptions { Timeout = 15000 }); - - _logger.LogInformation("Map canvas is ready - waiting for map to render"); - - // Wait for Esri map view to be ready - try - { - await page.WaitForFunctionAsync(@"() => { - try { - const elements = document.querySelectorAll('.esri-view'); - for (let el of elements) { - if (el.__view && el.__view.ready) { - return true; - } - } - } catch(e) {} - return false; - }", new PageWaitForFunctionOptions { Timeout = 30000 }); - _logger.LogInformation("Esri map view is ready"); - } - catch - { - _logger.LogInformation("Esri view ready check timed out, continuing with fixed wait"); - } - - // Additional wait for tiles to render - await page.WaitForTimeoutAsync(_tileRenderWaitMs); - - await _debugService.SaveStepDebugAsync(debugFolder, 7, "weather_map_ready", page, consoleMessages, networkRequests, cancellationToken); - - // Step 8: Ensure radar is paused before capturing frames - _logger.LogInformation("Checking if radar loop is paused"); - var playPauseButton = page.Locator("button[data-testid='bom-time-scrub-play-pause']").First; - await playPauseButton.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000 }); - - // Check if button shows "Play" (paused) or "Pause" (playing) - var buttonLabel = await playPauseButton.Locator(".bom-scrub-action__label").TextContentAsync(); - if (buttonLabel?.Trim().Equals("Pause", StringComparison.OrdinalIgnoreCase) == true) - { - _logger.LogInformation("Radar is playing, pausing it"); - await playPauseButton.ClickAsync(); - // Wait for pause to take effect - await page.WaitForTimeoutAsync(500); - - // Verify it's now paused - buttonLabel = await playPauseButton.Locator(".bom-scrub-action__label").TextContentAsync(); - if (buttonLabel?.Trim().Equals("Play", StringComparison.OrdinalIgnoreCase) != true) - { - _logger.LogWarning("Radar may not be paused after click, continuing anyway"); - } - } - else - { - _logger.LogInformation("Radar is already paused"); - } - - await _debugService.SaveStepDebugAsync(debugFolder, 8, "radar_paused", page, consoleMessages, networkRequests, cancellationToken); - - // Step 9: Click on first frame segment to ensure we start at frame 0 - _logger.LogInformation("Resetting to first frame (frame 0)"); - try - { - var firstFrameSegment = page.Locator("[data-testid='bom-scrub-segment'][data-id='0']").First; - await firstFrameSegment.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000 }); - await firstFrameSegment.ClickAsync(); - // Wait for frame to update - await page.WaitForTimeoutAsync(1000); - _logger.LogInformation("Successfully clicked frame 0 segment"); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to click first frame segment, continuing anyway"); - } - - await _debugService.SaveStepDebugAsync(debugFolder, 9, "frame_0_selected", page, consoleMessages, networkRequests, cancellationToken); - - // Step 10: Verify scrubber is at position 0 before capturing first frame - _logger.LogInformation("Verifying scrubber is at position 0"); - try - { - // Check if the active segment has data-id="0" (simpler and more reliable than checking thumb position) - var activeSegment = await page.EvaluateAsync(@"() => { - const segments = Array.from(document.querySelectorAll('[data-testid=""bom-scrub-segment""]')); - const activeSegment = segments.find(s => { - const style = window.getComputedStyle(s); - return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)'; - }); - return activeSegment && activeSegment.getAttribute('data-id') === '0'; - }"); - - if (activeSegment) - { - _logger.LogInformation("Scrubber confirmed at position 0 (active segment is frame 0)"); - } - else - { - _logger.LogDebug("Could not confirm scrubber position via active segment, but continuing (frame 0 was clicked)"); - } - } - catch (Exception ex) - { - _logger.LogDebug(ex, "Scrubber position verification failed, continuing anyway (non-critical check)"); - } - - await _debugService.SaveStepDebugAsync(debugFolder, 10, "scrubber_at_position_0", page, consoleMessages, networkRequests, cancellationToken); - - // Step 11: Wait for frame 0 tiles to fully load before calculating bounding box - // This ensures the map viewport is stable and prevents jiggle between frames - _logger.LogInformation("Waiting for frame 0 tiles to fully render"); - await page.WaitForTimeoutAsync(_tileRenderWaitMs); - - // Step 12: Extract metadata and frame information - _logger.LogInformation("Extracting metadata and frame information"); - var lastUpdatedInfo = await _timeParsingService.ExtractLastUpdatedInfoAsync(page); - var frameInfo = await ExtractFrameInfoAsync(page); - - // Step 13: Get map container and calculate bounding box once - // Calculate after frame 0 tiles are loaded to ensure consistent viewport - _logger.LogInformation("Preparing map container for screenshot"); - var mapContainer = page.Locator(".esri-view-surface").First; - await mapContainer.WaitForAsync(new LocatorWaitForOptions { Timeout = 10000 }); - - // Ensure the map container is visible and has dimensions - await page.WaitForFunctionAsync(@"() => { - const container = document.querySelector('.esri-view-surface'); - return container && container.offsetWidth > 0 && container.offsetHeight > 0; - }", new PageWaitForFunctionOptions { Timeout = 10000 }); - - var boundingBox = await mapContainer.BoundingBoxAsync(); - if (boundingBox == null || boundingBox.Width <= 0 || boundingBox.Height <= 0) - { - _logger.LogError("Map container has invalid bounds: {BoundingBox}", boundingBox); - throw new Exception($"Map container has invalid bounds: {boundingBox?.Width ?? 0}x{boundingBox?.Height ?? 0}"); - } - - // Convert BoundingBox to Clip for crop calculation - var containerClip = new Clip - { - X = boundingBox.X, - Y = boundingBox.Y, - Width = boundingBox.Width, - Height = boundingBox.Height - }; - - // Step 14: Use provided cache folder (already created by BomRadarService) - Directory.CreateDirectory(cacheFolderPath); - _logger.LogInformation("Using cache folder: {Path}", cacheFolderPath); - - // Step 15-21: Capture all frames - var frameCount = CacheHelper.GetFrameCountForDataType(_configuration, CachedDataType.Radar); - - // Track progress: map is ready, starting frame capture - _cacheService.RecordUpdateProgressByFolder(cacheFolderPath, CacheUpdatePhase.CapturingFrames, 0, frameCount); - - var frames = new List(); - var stepForwardButton = page.Locator("button[data-testid='bom-scrub-utils__right__step-forward']").First; - int? previousMinutesAgo = null; - - for (int frameIndex = 0; frameIndex < frameCount; frameIndex++) - { - _logger.LogInformation("Capturing frame {FrameIndex} of {FrameCount}", frameIndex, frameCount); - - // Wait for map to stabilize (tiles to load for current frame) - await page.WaitForTimeoutAsync(_tileRenderWaitMs); - - // Extract actual minutes ago from the display label (e.g., "17 minutes ago" -> 17) - var minutesAgo = await ExtractMinutesAgoFromDisplayAsync(page); - if (minutesAgo == null) - { - // Fallback to calculated value if extraction fails - var (_, defaultMinutesAgo) = frameInfo[frameIndex]; - minutesAgo = defaultMinutesAgo; - _logger.LogWarning("Failed to extract minutes from display label for frame {FrameIndex}, using default: {MinutesAgo}", frameIndex, minutesAgo); - } - - // Validate that minutesAgo is different from previous frame (unless it's frame 0) - if (frameIndex > 0 && previousMinutesAgo.HasValue && minutesAgo == previousMinutesAgo.Value) - { - _logger.LogWarning("Frame {FrameIndex} has same minutesAgo ({MinutesAgo}) as previous frame. Waiting for display to update...", frameIndex, minutesAgo); - // Wait for display label to change - await WaitForDisplayLabelToChangeAsync(page, previousMinutesAgo.Value); - // Re-extract after waiting - minutesAgo = await ExtractMinutesAgoFromDisplayAsync(page); - if (minutesAgo == null || minutesAgo == previousMinutesAgo.Value) - { - // Still same or failed - use calculated fallback - var (_, defaultMinutesAgo) = frameInfo[frameIndex]; - minutesAgo = defaultMinutesAgo; - _logger.LogWarning("Display label did not update for frame {FrameIndex}, using calculated default: {MinutesAgo}", frameIndex, minutesAgo); - } - } - - // Take screenshot with crop configuration - // Ensure radar subfolder exists - var radarFolder = FilePathHelper.GetDataTypeFolderPath(cacheFolderPath, CachedDataType.Radar); - if (!Directory.Exists(radarFolder)) - { - Directory.CreateDirectory(radarFolder); - } - - var framePath = FilePathHelper.GetFrameFilePath(cacheFolderPath, CachedDataType.Radar, frameIndex); - await CaptureMapScreenshotAsync(page, mapContainer, framePath, containerClip); - - frames.Add(new RadarFrame - { - FrameIndex = frameIndex, - ImagePath = framePath, - MinutesAgo = minutesAgo.Value - }); - - previousMinutesAgo = minutesAgo; - - _logger.LogInformation("Frame {FrameIndex} saved: {Path} ({MinutesAgo} minutes ago)", - frameIndex, framePath, minutesAgo.Value); - - // Track progress: frame captured - _cacheService.RecordUpdateProgressByFolder(cacheFolderPath, CacheUpdatePhase.CapturingFrames, frameIndex + 1, frameCount); - - // Save debug screenshot BEFORE clicking step forward - await _debugService.SaveStepDebugAsync(debugFolder, 15 + frameIndex, $"frame_{frameIndex}_captured", page, consoleMessages, networkRequests, cancellationToken); - - // If not the last frame, click step forward to prepare for next frame - if (frameIndex < frameCount - 1) - { - // Dismiss any modal overlays (BOM, reCAPTCHA, feedback forms) before clicking - await DismissModalOverlaysAsync(page); - - // Get current minutesAgo before clicking (to detect change) - var currentMinutesAgo = await ExtractMinutesAgoFromDisplayAsync(page); - - // Use force click to bypass any remaining overlays - await stepForwardButton.ClickAsync(new LocatorClickOptions { Force = true }); - - // Wait for display label to actually change (not just a fixed timeout) - if (currentMinutesAgo.HasValue) - { - await WaitForDisplayLabelToChangeAsync(page, currentMinutesAgo.Value); - } - else - { - // Fallback to fixed wait if we can't detect change - await page.WaitForTimeoutAsync(1000); - } - } - } - - _logger.LogInformation("All {FrameCount} frames captured successfully", frameCount); - - // Step 22: Save metadata and frame information - // Track progress: switching to saving phase - _cacheService.RecordUpdateProgressByFolder(cacheFolderPath, CacheUpdatePhase.Saving); - - await _cacheService.SaveMetadataAsync(cacheFolderPath, lastUpdatedInfo, cancellationToken); - await _cacheService.SaveFramesMetadataAsync(cacheFolderPath, CachedDataType.Radar, frames, cancellationToken); - - // Step 23: Return response with all frames - var cacheExpiresAt = lastUpdatedInfo.ObservationTime.AddMinutes(_cacheExpirationMinutes); - return ResponseBuilder.CreateRadarResponse(cacheFolderPath, frames, _cacheManagementCheckIntervalMinutes, lastUpdatedInfo, suburb, state, cacheIsValid: true, cacheExpiresAt: cacheExpiresAt, isUpdating: false); - } - catch (Exception ex) - { - // Save error debug info if debug is enabled - await _debugService.SaveErrorDebugAsync(debugFolder, $"Exception: {ex.Message}\n\nStackTrace:\n{ex.StackTrace}", page, consoleMessages, networkRequests, cancellationToken); - _logger.LogError(ex, "Error during radar screenshot capture"); - throw; - } - } - - /// - /// Calculates the crop area for screenshot based on configuration - /// - private Clip CalculateCropArea(Clip containerClip) - { - // Start with container's position plus offset - var x = containerClip.X + _cropConfig.X; - var y = containerClip.Y + _cropConfig.Y; - - // Calculate width: container width minus left offset (X) minus right offset - var width = Math.Max(0, containerClip.Width - _cropConfig.X - _cropConfig.RightOffset); - - // Calculate height (use configured or remaining height) - var height = _cropConfig.Height ?? Math.Max(0, containerClip.Height - _cropConfig.Y); - - // Validate bounds - if (x < containerClip.X || y < containerClip.Y) - { - _logger.LogWarning("Crop offset is outside container bounds, using container bounds"); - x = containerClip.X; - y = containerClip.Y; - } - - var maxWidth = containerClip.Width - (x - containerClip.X); - var maxHeight = containerClip.Height - (y - containerClip.Y); - - if (width > maxWidth) - { - _logger.LogWarning("Crop width exceeds container bounds, adjusting from {Requested} to {Max}", width, maxWidth); - width = maxWidth; - } - - if (height > maxHeight) - { - _logger.LogWarning("Crop height exceeds container bounds, adjusting from {Requested} to {Max}", height, maxHeight); - height = maxHeight; - } - - if (width <= 0 || height <= 0) - { - throw new Exception($"Invalid crop dimensions: {width}x{height}"); - } - - _logger.LogDebug("Crop area calculated: X={X}, Y={Y}, Width={Width}, Height={Height} (container: {ContainerX}, {ContainerY}, {ContainerWidth}x{ContainerHeight})", - x, y, width, height, containerClip.X, containerClip.Y, containerClip.Width, containerClip.Height); - - return new Clip - { - X = x, - Y = y, - Width = width, - Height = height + Page = page, + Suburb = suburb, + State = state, + CacheFolderPath = cacheFolderPath, + DebugFolder = debugFolder, + ConsoleMessages = consoleMessages, + NetworkRequests = networkRequests }; - } - - /// - /// Extracts frame information from timeline segments - /// - private async Task> ExtractFrameInfoAsync(IPage page) - { - try - { - var frameInfo = await page.EvaluateAsync(@"() => { - const segments = Array.from(document.querySelectorAll('[data-testid=""bom-scrub-segment""]')); - return segments.map((seg, index) => { - const ariaLabel = seg.getAttribute('aria-label') || ''; - // Extract minutes from '40 minutes ago', '35 minutes ago', etc. - const minutesMatch = ariaLabel.match(/(\d+)\s+minutes?\s+ago/); - const minutes = minutesMatch ? parseInt(minutesMatch[1]) : null; - return { index: index, minutesAgo: minutes }; - }); - }"); - - var result = new List<(int index, int minutesAgo)>(); - for (int i = 0; i < 7; i++) - { - // Default values if extraction fails - var minutesAgo = 40 - (i * 5); - - // Try to use extracted values if available - if (frameInfo != null && i < frameInfo.Length) - { - // The EvaluateAsync returns object[], we'd need to deserialize properly - // For now, use defaults - } - - result.Add((i, minutesAgo)); - } - - return result; - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to extract frame info, using defaults"); - // Return default frame info - return Enumerable.Range(0, 7) - .Select(i => (i, 40 - (i * 5))) - .ToList(); - } - } - - /// - /// Extracts the actual minutes ago value from the display label (e.g., "17 minutes ago" -> 17) - /// - private async Task ExtractMinutesAgoFromDisplayAsync(IPage page) - { - try - { - var timeLabel = await page.Locator(".bom-scrub-display-label").First.TextContentAsync(); - if (string.IsNullOrEmpty(timeLabel)) - { - return null; - } - - // Parse "17 minutes ago" or "41 minutes ago" etc. - var match = System.Text.RegularExpressions.Regex.Match( - timeLabel.Trim(), - @"(\d+)\s+minutes?\s+ago", - System.Text.RegularExpressions.RegexOptions.IgnoreCase - ); - - if (match.Success && match.Groups.Count >= 2) - { - if (int.TryParse(match.Groups[1].Value, out var minutes)) - { - return minutes; - } - } - - return null; - } - catch (Exception ex) - { - _logger.LogDebug(ex, "Failed to extract minutes from display label"); - return null; - } - } - - /// - /// Waits for the display label to change from the current minutesAgo value - /// - private async Task WaitForDisplayLabelToChangeAsync(IPage page, int currentMinutesAgo, int maxWaitMs = 5000) - { - try - { - var startTime = DateTime.UtcNow; - while ((DateTime.UtcNow - startTime).TotalMilliseconds < maxWaitMs) - { - var newMinutesAgo = await ExtractMinutesAgoFromDisplayAsync(page); - if (newMinutesAgo.HasValue && newMinutesAgo.Value != currentMinutesAgo) - { - // Display has changed - return; - } - await page.WaitForTimeoutAsync(200); // Check every 200ms - } - _logger.LogDebug("Display label did not change from {CurrentMinutesAgo} within {MaxWaitMs}ms", currentMinutesAgo, maxWaitMs); - } - catch (Exception ex) - { - _logger.LogDebug(ex, "Error waiting for display label to change"); - } - } - - /// - /// Dismisses any modal overlays (BOM modals, reCAPTCHA, feedback forms) that might block the radar - /// Uses a single efficient check and minimal delays - /// - private async Task DismissModalOverlaysAsync(IPage page) - { - try - { - // Single check for all modal types (BOM, reCAPTCHA, feedback forms) - var hasModal = await page.EvaluateAsync(@"() => { - // Check BOM modal overlay - const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); - if (bomOverlay && bomOverlay.style.display !== 'none') { - return true; - } - - // Check reCAPTCHA modals (large visible elements) - const recaptchaSelectors = [ - '.g-recaptcha', - '#g-recaptcha', - '.rc-anchor-container' - ]; - for (const selector of recaptchaSelectors) { - const el = document.querySelector(selector); - if (el) { - const style = window.getComputedStyle(el); - const rect = el.getBoundingClientRect(); - if (style.display !== 'none' && style.visibility !== 'hidden' && - rect.width > 200 && rect.height > 200) { - return true; - } - } - } - - // Check feedback forms with reCAPTCHA - const allForms = document.querySelectorAll('form'); - for (const form of allForms) { - const action = form.getAttribute('action') || ''; - const id = form.getAttribute('id') || ''; - if (action.includes('feedback') || id.includes('feedback')) { - const style = window.getComputedStyle(form); - const rect = form.getBoundingClientRect(); - if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { - const text = form.textContent || ''; - if (text.includes('reCAPTCHA') || text.includes('recaptcha') || text.includes('Tell us why')) { - return true; - } - } - } - } - - return false; - }"); - - if (!hasModal) - { - return; // No modal, exit quickly - } - - _logger.LogDebug("Modal overlay detected, dismissing"); - - // Single Escape key press (works for most modals) - await page.Keyboard.PressAsync("Escape"); - await page.WaitForTimeoutAsync(300); // Minimal delay - - // If still visible, try clicking outside (on map) - var stillVisible = await page.EvaluateAsync(@"() => { - const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); - if (bomOverlay && bomOverlay.style.display !== 'none') return true; - - // Check feedback forms - const allForms = document.querySelectorAll('form'); - for (const form of allForms) { - const action = form.getAttribute('action') || ''; - const id = form.getAttribute('id') || ''; - if (action.includes('feedback') || id.includes('feedback')) { - const style = window.getComputedStyle(form); - const rect = form.getBoundingClientRect(); - if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { - return true; - } - } - } - return false; - }"); - - if (stillVisible) - { - // Click on map to dismiss (only if Escape didn't work) - try - { - var mapContainer = page.Locator(".esri-view-surface").First; - await mapContainer.ClickAsync(new LocatorClickOptions { Force = true }); - await page.WaitForTimeoutAsync(200); // Minimal delay - } - catch - { - // Ignore if click fails - } - } - } - catch (Exception ex) - { - _logger.LogDebug(ex, "Error dismissing modal overlay, continuing"); - } - } - - /// - /// Captures map screenshot with crop configuration - /// - private async Task CaptureMapScreenshotAsync(IPage page, ILocator mapContainer, string outputPath, Clip containerClip) - { - // First, validate container clip itself - if (containerClip == null || containerClip.Width <= 0 || containerClip.Height <= 0) - { - _logger.LogError("Invalid container bounds: X={X}, Y={Y}, Width={Width}, Height={Height}", - containerClip?.X ?? 0, containerClip?.Y ?? 0, containerClip?.Width ?? 0, containerClip?.Height ?? 0); - throw new Exception($"Invalid container bounds: {containerClip?.Width ?? 0}x{containerClip?.Height ?? 0}"); - } - Clip cropArea; - try - { - cropArea = CalculateCropArea(containerClip); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to calculate crop area, using full container bounds. Container: X={X}, Y={Y}, Width={Width}, Height={Height}", - containerClip.X, containerClip.Y, containerClip.Width, containerClip.Height); - // Fallback to full container if crop calculation fails - cropArea = containerClip; - } + // Get the appropriate workflow (currently only RadarScraping is implemented) + var workflow = _workflowFactory.GetWorkflow("RadarScraping"); - // Get viewport size - if null, try to get it from page evaluation as fallback - var viewportSize = page.ViewportSize; - int? viewportWidth = viewportSize?.Width; - int? viewportHeight = viewportSize?.Height; + _logger.LogInformation("Executing {Workflow} workflow for {Suburb}, {State}", workflow.Name, suburb, state); - if (viewportWidth == null || viewportHeight == null) - { - try - { - var viewportJson = await page.EvaluateAsync("() => JSON.stringify({ width: window.innerWidth, height: window.innerHeight })"); - if (!string.IsNullOrEmpty(viewportJson)) - { - using var doc = System.Text.Json.JsonDocument.Parse(viewportJson); - var root = doc.RootElement; - if (root.TryGetProperty("width", out var widthProp) && root.TryGetProperty("height", out var heightProp)) - { - if (widthProp.TryGetInt32(out var width) && heightProp.TryGetInt32(out var height)) - { - viewportWidth = width; - viewportHeight = height; - _logger.LogDebug("Retrieved viewport size from page evaluation: {Width}x{Height}", width, height); - } - } - } - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to get viewport size from page evaluation"); - } - - // If still can't get it, use container bounds as fallback for validation - if (viewportWidth == null || viewportHeight == null) - { - _logger.LogWarning("Cannot determine viewport size, using container bounds for validation"); - viewportWidth = (int)containerClip.Width; - viewportHeight = (int)containerClip.Height; - } - } - - // Validate crop area is within page bounds and adjust if necessary - if (viewportWidth.HasValue && viewportHeight.HasValue) - { - // Ensure crop area coordinates are non-negative - if (cropArea.X < 0) - { - _logger.LogWarning("Crop X is negative ({X}), adjusting to 0", cropArea.X); - cropArea = new Clip { X = 0, Y = cropArea.Y, Width = cropArea.Width + cropArea.X, Height = cropArea.Height }; - } - if (cropArea.Y < 0) - { - _logger.LogWarning("Crop Y is negative ({Y}), adjusting to 0", cropArea.Y); - cropArea = new Clip { X = cropArea.X, Y = 0, Width = cropArea.Width, Height = cropArea.Height + cropArea.Y }; - } - - // Ensure crop area doesn't exceed viewport bounds - if (cropArea.X + cropArea.Width > viewportWidth.Value) - { - var newWidth = viewportWidth.Value - cropArea.X; - _logger.LogWarning("Crop width exceeds viewport ({Requested} > {Max}), adjusting to {NewWidth}", - cropArea.Width, viewportWidth.Value, newWidth); - cropArea = new Clip { X = cropArea.X, Y = cropArea.Y, Width = newWidth, Height = cropArea.Height }; - } - if (cropArea.Y + cropArea.Height > viewportHeight.Value) - { - var newHeight = viewportHeight.Value - cropArea.Y; - _logger.LogWarning("Crop height exceeds viewport ({Requested} > {Max}), adjusting to {NewHeight}", - cropArea.Height, viewportHeight.Value, newHeight); - cropArea = new Clip { X = cropArea.X, Y = cropArea.Y, Width = cropArea.Width, Height = newHeight }; - } - } - - // Final validation - ensure dimensions are positive - if (cropArea.Width <= 0 || cropArea.Height <= 0) - { - _logger.LogError("Invalid crop dimensions after validation: {Width}x{Height}, using full container", cropArea.Width, cropArea.Height); - cropArea = containerClip; - } - - // Double-check container clip is still valid as final fallback - if (cropArea.Width <= 0 || cropArea.Height <= 0) - { - _logger.LogError("Cannot create valid crop area. Container: {ContainerWidth}x{ContainerHeight}, Viewport: {ViewportWidth}x{ViewportHeight}", - containerClip.Width, containerClip.Height, viewportWidth ?? 0, viewportHeight ?? 0); - throw new Exception($"Cannot create valid crop area. Container: {containerClip.Width}x{containerClip.Height}, Viewport: {viewportWidth ?? 0}x{viewportHeight ?? 0}"); - } - - // Wait for fonts to be loaded to prevent text rendering artifacts - try - { - await page.WaitForLoadStateAsync(LoadState.NetworkIdle, new PageWaitForLoadStateOptions { Timeout = 5000 }); - } - catch - { - // Continue if network idle timeout - fonts may already be loaded - } - - // Dismiss any modal overlays (BOM, reCAPTCHA, feedback forms) before taking screenshot - await DismissModalOverlaysAsync(page); - - // Take high-quality screenshot with explicit PNG format and disabled animations - await page.ScreenshotAsync(new PageScreenshotOptions - { - Path = outputPath, - Clip = cropArea, - Type = ScreenshotType.Png, // Explicit PNG for lossless quality - Animations = ScreenshotAnimations.Disabled // Disable animations to prevent artifacts - }); - - _logger.LogDebug("Screenshot saved: {Path} (crop: {X},{Y} {Width}x{Height})", - outputPath, cropArea.X, cropArea.Y, cropArea.Width, cropArea.Height); + return await workflow.ExecuteAsync(context, cancellationToken); } } - diff --git a/Services/SelectorService.cs b/Services/SelectorService.cs new file mode 100644 index 0000000..e82ee04 --- /dev/null +++ b/Services/SelectorService.cs @@ -0,0 +1,60 @@ +using BomLocalService.Models; +using BomLocalService.Services.Interfaces; +using Microsoft.Playwright; + +namespace BomLocalService.Services; + +public class SelectorService : ISelectorService +{ + private readonly ILogger _logger; + + public SelectorService(ILogger logger) + { + _logger = logger; + } + + public async Task FindElementAsync(IPage page, SelectorConfig config, CancellationToken cancellationToken = default) + { + foreach (var selector in config.Selectors) + { + try + { + var locator = page.Locator(selector).First; + if (await locator.IsVisibleAsync()) + { + _logger.LogInformation("Found {Name} with selector: {Selector}", config.Name, selector); + return locator; + } + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Selector {Selector} not found or not visible for {Name}", selector, config.Name); + continue; + } + } + + if (config.Required) + { + var errorMsg = config.ErrorMessage ?? $"Required element {config.Name} not found with any selector"; + _logger.LogWarning("{Message}", errorMsg); + } + else + { + _logger.LogDebug("Optional element {Name} not found with any selector", config.Name); + } + + return null; + } + + public ILocator GetLocator(IPage page, SelectorConfig config) + { + if (config.Selectors.Length == 0) + { + throw new ArgumentException($"No selectors configured for {config.Name}"); + } + + // Returns first selector as locator (for cases where we know it exists) + return page.Locator(config.Selectors[0]).First; + } +} + diff --git a/appsettings.json b/appsettings.json index 63be0d2..debaea5 100644 --- a/appsettings.json +++ b/appsettings.json @@ -25,8 +25,8 @@ "IntervalHours": 1 }, "Screenshot": { - "DynamicContentWaitMs": 2000, - "TileRenderWaitMs": 5000, + "DynamicContentWaitMs": 1500, + "TileRenderWaitMs": 3000, "Crop": { "X": 250, "Y": 0, @@ -47,5 +47,200 @@ "TimeSeries": { "WarningFolderCount": 200, "MaxTimeRangeHours": null + }, + "Scraping": { + "BaseUrl": "https://www.bom.gov.au/", + "Timeouts": { + "PageNavigation": 30000, + "ElementWait": 10000, + "SearchResults": 10000, + "MapRender": 15000, + "TileRender": 5000 + }, + "Selectors": { + "SearchButton": { + "Name": "Search Button", + "Selectors": [ + "button[data-testid='searchLabel']", + "button[aria-label='Search for a location']", + "button.search-location__trigger-button", + "button.bom-button:has(span.bom-button__label:has-text('Search for a location'))", + "button:has-text('Search for a location')", + "[data-testid='showSearchModal'] button" + ], + "TimeoutMs": 10000, + "Required": true, + "ErrorMessage": "Could not find 'Search for a location' button on BOM homepage." + }, + "SearchInput": { + "Name": "Search Input", + "Selectors": ["#search-enter-keyword"], + "TimeoutMs": 5000, + "Required": true + }, + "SearchResultsList": { + "Name": "Search Results List", + "Selectors": ["ul[aria-labelledby='location-results-title']"], + "TimeoutMs": 10000, + "Required": true + }, + "SearchResultItem": { + "Name": "Search Result Item", + "Selectors": ["li.bom-linklist__item[role='listitem']"], + "TimeoutMs": 5000, + "Required": true + }, + "LocationName": { + "Name": "Location Name", + "Selectors": ["[data-testid='location-name']"], + "TimeoutMs": 5000, + "Required": false + }, + "LocationDescription": { + "Name": "Location Description", + "Selectors": [".bom-linklist-item__desc"], + "TimeoutMs": 5000, + "Required": false + }, + "ResultsTitle": { + "Name": "Results Title", + "Selectors": ["#location-results-title", "[data-testid='location-results-title']"], + "TimeoutMs": 5000, + "Required": false + }, + "RadarLink": { + "Name": "Radar Link", + "Selectors": [ + "a.jump-link.bom-button.bom-button--secondary:has-text('Rain radar and weather map')", + ".cta.button a.jump-link:has-text('Rain radar and weather map')", + "a.jump-link:has-text('Rain radar and weather map')", + ".cta.button a:has-text('Rain radar and weather map')", + "a.bom-button--secondary:has-text('Rain radar and weather map')", + "a:has-text('Rain radar and weather map')", + "a:has-text('rain radar')", + "a[href*='radar']" + ], + "TimeoutMs": 10000, + "Required": true, + "ErrorMessage": "Could not find 'Rain radar and weather map' link" + }, + "MapCanvas": { + "Name": "Map Canvas", + "Selectors": [".esri-view-surface canvas"], + "TimeoutMs": 15000, + "Required": true + }, + "MapContainer": { + "Name": "Map Container", + "Selectors": [".esri-view-surface"], + "TimeoutMs": 10000, + "Required": true + }, + "PlayPauseButton": { + "Name": "Play Pause Button", + "Selectors": ["button[data-testid='bom-time-scrub-play-pause']"], + "TimeoutMs": 5000, + "Required": true + }, + "PlayPauseLabel": { + "Name": "Play Pause Label", + "Selectors": [".bom-scrub-action__label"], + "TimeoutMs": 2000, + "Required": false + }, + "FrameSegment": { + "Name": "Frame Segment", + "Selectors": ["[data-testid='bom-scrub-segment']"], + "TimeoutMs": 5000, + "Required": false + }, + "StepForwardButton": { + "Name": "Step Forward Button", + "Selectors": ["button[data-testid='bom-scrub-utils__right__step-forward']"], + "TimeoutMs": 5000, + "Required": true + }, + "TimeDisplayLabel": { + "Name": "Time Display Label", + "Selectors": [".bom-scrub-display-label"], + "TimeoutMs": 2000, + "Required": false + }, + "ModalOverlay": { + "Name": "Modal Overlay", + "Selectors": [".bom-modal-overlay--after-open"], + "TimeoutMs": 1000, + "Required": false + }, + "WeatherMetadata": { + "Name": "Weather Metadata", + "Selectors": ["section[data-testid='weatherMetadata']", "section[aria-label='Last updated']"], + "TimeoutMs": 5000, + "Required": false + } + }, + "JavaScriptTemplates": { + "WaitForSearchResults": "() => { const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=\"listitem\"]')); return results.length > 0 && results.some(r => r.offsetParent !== null); }", + "ExtractSearchResults": "() => { const resultsList = document.querySelector('ul[aria-labelledby=\"location-results-title\"]'); if (!resultsList) { console.log('Location results list not found'); return []; } const results = Array.from(resultsList.querySelectorAll('li.bom-linklist__item[role=\"listitem\"]')); console.log('Found', results.length, 'location results'); return results.map((r) => { const nameEl = r.querySelector('[data-testid=\"location-name\"]'); const descEl = r.querySelector('.bom-linklist-item__desc'); const name = nameEl ? (nameEl.textContent || nameEl.innerText || '').trim() : ''; const desc = descEl ? (descEl.textContent || descEl.innerText || '').trim() : ''; const fullText = (r.textContent || r.innerText || '').trim(); console.log('Result:', { hasNameEl: !!nameEl, hasDescEl: !!descEl, name: name, desc: desc }); return [name, desc, fullText]; }); }", + "ExtractSearchResultsFallback": "() => { const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=\"listitem\"]')); return results.map(r => r.textContent || ''); }", + "WaitForMapCanvas": "() => { const canvas = document.querySelector('.esri-view-surface canvas'); return canvas && canvas.width > 0 && canvas.height > 0 && canvas.offsetWidth > 0 && canvas.offsetHeight > 0; }", + "WaitForEsriView": "() => { try { const elements = document.querySelectorAll('.esri-view'); for (let el of elements) { if (el.__view && el.__view.ready) { return true; } } } catch(e) {} return false; }", + "CheckActiveFrameSegment": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); const activeSegment = segments.find(s => { const style = window.getComputedStyle(s); return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)'; }); return activeSegment && activeSegment.getAttribute('data-id') === '0'; }", + "WaitForMapContainer": "() => { const container = document.querySelector('.esri-view-surface'); return container && container.offsetWidth > 0 && container.offsetHeight > 0; }", + "ExtractFrameInfo": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); const now = new Date(); return segments.map((seg, index) => { const ariaLabel = seg.getAttribute('aria-label') || ''; let minutes = null; const timestampMatch = ariaLabel.match(/(?:[A-Za-z]+\\s+)?(\\d{1,2})\\s+([A-Za-z]{3}),?\\s+(\\d{1,2}):(\\d{2})\\s+(am|pm)/i); if (timestampMatch) { try { const day = parseInt(timestampMatch[1]); const monthStr = timestampMatch[2]; const hour12 = parseInt(timestampMatch[3]); const minute = parseInt(timestampMatch[4]); const ampm = timestampMatch[5].toLowerCase(); const monthNames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']; const month = monthNames.indexOf(monthStr.toLowerCase()); if (month >= 0) { let hour24 = hour12; if (ampm === 'pm' && hour12 !== 12) hour24 += 12; if (ampm === 'am' && hour12 === 12) hour24 = 0; const year = now.getFullYear(); const frameTime = new Date(year, month, day, hour24, minute); if (frameTime > now) frameTime.setFullYear(year - 1); const diffMs = now - frameTime; minutes = Math.round(diffMs / (1000 * 60)); if (minutes < 0 || minutes > 120) minutes = null; } } catch(e) {} } return { index: index, minutesAgo: minutes }; }); }", + "CheckModalOverlay": "() => { const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); if (bomOverlay && bomOverlay.style.display !== 'none') { return true; } const recaptchaSelectors = ['.g-recaptcha', '#g-recaptcha', '.rc-anchor-container']; for (const selector of recaptchaSelectors) { const el = document.querySelector(selector); if (el) { const style = window.getComputedStyle(el); const rect = el.getBoundingClientRect(); if (style.display !== 'none' && style.visibility !== 'hidden' && rect.width > 200 && rect.height > 200) { return true; } } } const allForms = document.querySelectorAll('form'); for (const form of allForms) { const action = form.getAttribute('action') || ''; const id = form.getAttribute('id') || ''; if (action.includes('feedback') || id.includes('feedback')) { const style = window.getComputedStyle(form); const rect = form.getBoundingClientRect(); if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { const text = form.textContent || ''; if (text.includes('reCAPTCHA') || text.includes('recaptcha') || text.includes('Tell us why')) { return true; } } } } return false; }", + "CheckModalStillVisible": "() => { const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); if (bomOverlay && bomOverlay.style.display !== 'none') return true; const allForms = document.querySelectorAll('form'); for (const form of allForms) { const action = form.getAttribute('action') || ''; const id = form.getAttribute('id') || ''; if (action.includes('feedback') || id.includes('feedback')) { const style = window.getComputedStyle(form); const rect = form.getBoundingClientRect(); if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { return true; } } } return false; }", + "GetViewportSize": "() => JSON.stringify({ width: window.innerWidth, height: window.innerHeight })", + "ExtractWeatherMetadata": "() => { const section = document.querySelector('section[data-testid=\"weatherMetadata\"]') || document.querySelector('section[aria-label=\"Last updated\"]'); if (!section) return null; const divs = section.querySelectorAll('div'); return Array.from(divs).map(div => div.textContent.trim()).filter(text => text).join(' '); }" + }, + "TextPatterns": { + "ResultsCountPattern": "(\\d+)\\s+of\\s+(\\d+)", + "TimestampPattern": "(?:[A-Za-z]+\\s+)?\\d{1,2}\\s+[A-Za-z]{3},?\\s+\\d{1,2}:\\d{2}\\s+(?:am|pm)", + "ObservationTimePattern": "Observations:\\s*(\\d+)\\s*minutes?\\s*ago", + "ForecastTimePattern": "Forecast:\\s*(\\d+)\\s*minutes?\\s+ago", + "ForecastHourAgoPattern": "Forecast:\\s*an\\s+hour\\s+ago", + "WeatherStationPattern": "at\\s+([^,]+)\\s+weather\\s+station", + "DistancePattern": "(\\d+)\\s*km\\s+from", + "ExpectedTexts": { + "PlayButtonLabel": "Play", + "PauseButtonLabel": "Pause", + "RadarLinkText": "Rain radar and weather map" + } + }, + "Workflows": { + "RadarScraping": { + "Description": "Scrapes radar images for a location", + "Steps": { + "NavigateHomepage": { "Enabled": true }, + "ClickSearchButton": { "Enabled": true }, + "FillSearchInput": { "Enabled": true }, + "WaitForSearchResults": { "Enabled": true }, + "SelectSearchResult": { "Enabled": true }, + "ClickRadarLink": { "Enabled": true }, + "WaitForMapReady": { "Enabled": true }, + "PauseRadar": { "Enabled": true }, + "ResetToFirstFrame": { "Enabled": true }, + "ExtractMetadata": { "Enabled": true }, + "CalculateMapBounds": { "Enabled": true }, + "CaptureFrames": { + "Enabled": true, + "Parameters": { + "FrameCount": 7, + "WaitBetweenFrames": 5000 + } + } + } + }, + "TemperatureMap": { + "Description": "Scrapes temperature forecast map (future feature)", + "Steps": { + "NavigateHomepage": { "Enabled": true }, + "ClickSearchButton": { "Enabled": true }, + "FillSearchInput": { "Enabled": true }, + "WaitForSearchResults": { "Enabled": true }, + "SelectSearchResult": { "Enabled": true } + } + } + } } }