mirror of
https://github.com/alexhopeoconnor/bom-local-service.git
synced 2026-10-03 21:21:59 +10:00
Refactor scraping system to workflow-based architecture
Major architectural changes: - Replace monolithic ScrapingService with workflow-based system - Extract scraping logic into discrete, testable steps - Add configuration-driven selectors, JavaScript templates, and text patterns - Implement step registry and workflow factory for extensibility - Add PageState tracking and step dependencies - Make IWorkflow generic to support different response types New components: - Scraping steps organized by category (Navigation, Search, Map, Metadata, Capture) - BaseScrapingStep base class with common dependencies - ScrapingContext for shared state between steps - RadarScrapingWorkflow orchestrating step execution - SelectorService for configurable element selection - Configuration models (SelectorConfig, TextPatternsConfig, JavaScriptTemplatesConfig) Performance and monitoring: - Add step-level timing metrics with historical tracking - Log step durations and compare to averages - Warn on slow steps/workflows (50%+ slower than average) - Integrate metrics with CacheService Configuration improvements: - Externalize all selectors to appsettings.json - Externalize JavaScript templates to appsettings.json - Externalize text patterns to appsettings.json - Support Docker configuration via mounted appsettings.json - Remove hardcoded waits, use configurable timeouts Bug fixes: - Fix DebugService collection modification exception - Fix ResetToFirstFrameStep click interception with JavaScript fallback - Remove minutes ago fallback, use timestamp parsing only - Update timestamp parsing for new BOM website format Documentation: - Update README with new architecture details - Add configuration section with Docker guidance - Add development guidelines for extending scraping system - Add performance monitoring documentation
This commit is contained in:
+4
-1
@@ -24,7 +24,7 @@ RUN apt-get update && \
|
||||
chmod +x dotnet-install.sh && \
|
||||
./dotnet-install.sh --channel 9.0 --install-dir /usr/share/dotnet && \
|
||||
rm dotnet-install.sh && \
|
||||
apt-get install -y xvfb x11vnc fluxbox && \
|
||||
apt-get install -y xvfb x11vnc fluxbox curl && \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
@@ -45,4 +45,7 @@ LABEL org.opencontainers.image.licenses=MIT
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
|
||||
CMD curl --fail http://localhost:8080/api/health || exit 1
|
||||
|
||||
ENTRYPOINT ["/app/start.sh"]
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
namespace BomLocalService.Models;
|
||||
|
||||
/// <summary>
|
||||
/// JavaScript code templates used for page evaluation
|
||||
/// </summary>
|
||||
public class JavaScriptTemplatesConfig
|
||||
{
|
||||
public string WaitForSearchResults { get; set; } = @"() => {
|
||||
const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=""listitem""]'));
|
||||
return results.length > 0 && results.some(r => r.offsetParent !== null);
|
||||
}";
|
||||
|
||||
public string ExtractSearchResults { get; set; } = @"() => {
|
||||
const resultsList = document.querySelector('ul[aria-labelledby=""location-results-title""]');
|
||||
if (!resultsList) {
|
||||
console.log('Location results list not found');
|
||||
return [];
|
||||
}
|
||||
const results = Array.from(resultsList.querySelectorAll('li.bom-linklist__item[role=""listitem""]'));
|
||||
console.log('Found', results.length, 'location results');
|
||||
return results.map((r) => {
|
||||
const nameEl = r.querySelector('[data-testid=""location-name""]');
|
||||
const descEl = r.querySelector('.bom-linklist-item__desc');
|
||||
const name = nameEl ? (nameEl.textContent || nameEl.innerText || '').trim() : '';
|
||||
const desc = descEl ? (descEl.textContent || descEl.innerText || '').trim() : '';
|
||||
const fullText = (r.textContent || r.innerText || '').trim();
|
||||
console.log('Result:', { hasNameEl: !!nameEl, hasDescEl: !!descEl, name: name, desc: desc });
|
||||
return [name, desc, fullText];
|
||||
});
|
||||
}";
|
||||
|
||||
public string ExtractSearchResultsFallback { get; set; } = @"() => {
|
||||
const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=""listitem""]'));
|
||||
return results.map(r => r.textContent || '');
|
||||
}";
|
||||
|
||||
public string WaitForMapCanvas { get; set; } = @"() => {
|
||||
const canvas = document.querySelector('.esri-view-surface canvas');
|
||||
return canvas && canvas.width > 0 && canvas.height > 0 && canvas.offsetWidth > 0 && canvas.offsetHeight > 0;
|
||||
}";
|
||||
|
||||
public string WaitForEsriView { get; set; } = @"() => {
|
||||
try {
|
||||
const elements = document.querySelectorAll('.esri-view');
|
||||
for (let el of elements) {
|
||||
if (el.__view && el.__view.ready) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
} catch(e) {}
|
||||
return false;
|
||||
}";
|
||||
|
||||
public string CheckActiveFrameSegment { get; set; } = @"() => {
|
||||
const segments = Array.from(document.querySelectorAll('[data-testid=""bom-scrub-segment""]'));
|
||||
const activeSegment = segments.find(s => {
|
||||
const style = window.getComputedStyle(s);
|
||||
return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)';
|
||||
});
|
||||
return activeSegment && activeSegment.getAttribute('data-id') === '0';
|
||||
}";
|
||||
|
||||
public string WaitForMapContainer { get; set; } = @"() => {
|
||||
const container = document.querySelector('.esri-view-surface');
|
||||
return container && container.offsetWidth > 0 && container.offsetHeight > 0;
|
||||
}";
|
||||
|
||||
public string ExtractFrameInfo { get; set; } = @"() => {
|
||||
const segments = Array.from(document.querySelectorAll('[data-testid=""bom-scrub-segment""]'));
|
||||
const now = new Date();
|
||||
return segments.map((seg, index) => {
|
||||
const ariaLabel = seg.getAttribute('aria-label') || '';
|
||||
let minutes = null;
|
||||
|
||||
// Parse timestamp format: ""Wednesday 17 Dec, 11:05 pm"" or ""17 Dec, 11:05 pm""
|
||||
const timestampMatch = ariaLabel.match(/(?:[A-Za-z]+\s+)?(\d{1,2})\s+([A-Za-z]{3}),?\s+(\d{1,2}):(\d{2})\s+(am|pm)/i);
|
||||
if (timestampMatch) {
|
||||
try {
|
||||
const day = parseInt(timestampMatch[1]);
|
||||
const monthStr = timestampMatch[2];
|
||||
const hour12 = parseInt(timestampMatch[3]);
|
||||
const minute = parseInt(timestampMatch[4]);
|
||||
const ampm = timestampMatch[5].toLowerCase();
|
||||
|
||||
const monthNames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];
|
||||
const month = monthNames.indexOf(monthStr.toLowerCase());
|
||||
|
||||
if (month >= 0) {
|
||||
let hour24 = hour12;
|
||||
if (ampm === 'pm' && hour12 !== 12) hour24 += 12;
|
||||
if (ampm === 'am' && hour12 === 12) hour24 = 0;
|
||||
|
||||
const year = now.getFullYear();
|
||||
const frameTime = new Date(year, month, day, hour24, minute);
|
||||
|
||||
// If parsed time is in future, assume it's from last year
|
||||
if (frameTime > now) {
|
||||
frameTime.setFullYear(year - 1);
|
||||
}
|
||||
|
||||
const diffMs = now - frameTime;
|
||||
minutes = Math.round(diffMs / (1000 * 60));
|
||||
|
||||
// Validate reasonable range (0-2 hours)
|
||||
if (minutes < 0 || minutes > 120) {
|
||||
minutes = null;
|
||||
}
|
||||
}
|
||||
} catch(e) {
|
||||
// Parse error, leave minutes as null
|
||||
}
|
||||
}
|
||||
|
||||
return { index: index, minutesAgo: minutes };
|
||||
});
|
||||
}";
|
||||
|
||||
public string CheckModalOverlay { get; set; } = @"() => {
|
||||
const bomOverlay = document.querySelector('.bom-modal-overlay--after-open');
|
||||
if (bomOverlay && bomOverlay.style.display !== 'none') {
|
||||
return true;
|
||||
}
|
||||
|
||||
const recaptchaSelectors = ['.g-recaptcha', '#g-recaptcha', '.rc-anchor-container'];
|
||||
for (const selector of recaptchaSelectors) {
|
||||
const el = document.querySelector(selector);
|
||||
if (el) {
|
||||
const style = window.getComputedStyle(el);
|
||||
const rect = el.getBoundingClientRect();
|
||||
if (style.display !== 'none' && style.visibility !== 'hidden' &&
|
||||
rect.width > 200 && rect.height > 200) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const allForms = document.querySelectorAll('form');
|
||||
for (const form of allForms) {
|
||||
const action = form.getAttribute('action') || '';
|
||||
const id = form.getAttribute('id') || '';
|
||||
if (action.includes('feedback') || id.includes('feedback')) {
|
||||
const style = window.getComputedStyle(form);
|
||||
const rect = form.getBoundingClientRect();
|
||||
if (style.display !== 'none' && rect.width > 200 && rect.height > 200) {
|
||||
const text = form.textContent || '';
|
||||
if (text.includes('reCAPTCHA') || text.includes('recaptcha') || text.includes('Tell us why')) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}";
|
||||
|
||||
public string CheckModalStillVisible { get; set; } = @"() => {
|
||||
const bomOverlay = document.querySelector('.bom-modal-overlay--after-open');
|
||||
if (bomOverlay && bomOverlay.style.display !== 'none') return true;
|
||||
|
||||
const allForms = document.querySelectorAll('form');
|
||||
for (const form of allForms) {
|
||||
const action = form.getAttribute('action') || '';
|
||||
const id = form.getAttribute('id') || '';
|
||||
if (action.includes('feedback') || id.includes('feedback')) {
|
||||
const style = window.getComputedStyle(form);
|
||||
const rect = form.getBoundingClientRect();
|
||||
if (style.display !== 'none' && rect.width > 200 && rect.height > 200) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}";
|
||||
|
||||
public string GetViewportSize { get; set; } = @"() => JSON.stringify({ width: window.innerWidth, height: window.innerHeight })";
|
||||
|
||||
public string ExtractWeatherMetadata { get; set; } = @"() => {
|
||||
const section = document.querySelector('section[data-testid=""weatherMetadata""]') ||
|
||||
document.querySelector('section[aria-label=""Last updated""]');
|
||||
if (!section) return null;
|
||||
|
||||
const divs = section.querySelectorAll('div');
|
||||
return Array.from(divs).map(div => div.textContent.trim()).filter(text => text).join(' ');
|
||||
}";
|
||||
}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
namespace BomLocalService.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for all selectors used in scraping
|
||||
/// </summary>
|
||||
public class ScrapingSelectorsConfig
|
||||
{
|
||||
public SelectorConfig SearchButton { get; set; } = new();
|
||||
public SelectorConfig SearchInput { get; set; } = new();
|
||||
public SelectorConfig SearchResultsList { get; set; } = new();
|
||||
public SelectorConfig SearchResultItem { get; set; } = new();
|
||||
public SelectorConfig LocationName { get; set; } = new();
|
||||
public SelectorConfig LocationDescription { get; set; } = new();
|
||||
public SelectorConfig ResultsTitle { get; set; } = new();
|
||||
public SelectorConfig RadarLink { get; set; } = new();
|
||||
public SelectorConfig MapCanvas { get; set; } = new();
|
||||
public SelectorConfig MapContainer { get; set; } = new();
|
||||
public SelectorConfig PlayPauseButton { get; set; } = new();
|
||||
public SelectorConfig PlayPauseLabel { get; set; } = new();
|
||||
public SelectorConfig FrameSegment { get; set; } = new();
|
||||
public SelectorConfig StepForwardButton { get; set; } = new();
|
||||
public SelectorConfig TimeDisplayLabel { get; set; } = new();
|
||||
public SelectorConfig ModalOverlay { get; set; } = new();
|
||||
public SelectorConfig WeatherMetadata { get; set; } = new();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
namespace BomLocalService.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for a single scraping step
|
||||
/// </summary>
|
||||
public class ScrapingStepConfig
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether this step is enabled (can be disabled without removing from workflow)
|
||||
/// </summary>
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Optional parameters specific to this step
|
||||
/// </summary>
|
||||
public Dictionary<string, object>? Parameters { get; set; }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace BomLocalService.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for a scraping workflow
|
||||
/// </summary>
|
||||
public class ScrapingWorkflowConfig
|
||||
{
|
||||
public string Description { get; set; } = string.Empty;
|
||||
public Dictionary<string, ScrapingStepConfig> Steps { get; set; } = new();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
namespace BomLocalService.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for a single selector with multiple fallback options
|
||||
/// </summary>
|
||||
public class SelectorConfig
|
||||
{
|
||||
/// <summary>
|
||||
/// Human-readable name for this selector (for logging/debugging)
|
||||
/// </summary>
|
||||
public string Name { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Array of CSS selectors to try in order (first match wins)
|
||||
/// </summary>
|
||||
public string[] Selectors { get; set; } = Array.Empty<string>();
|
||||
|
||||
/// <summary>
|
||||
/// Timeout in milliseconds when waiting for this element
|
||||
/// </summary>
|
||||
public int TimeoutMs { get; set; } = 5000;
|
||||
|
||||
/// <summary>
|
||||
/// Whether this element is required (throws if not found)
|
||||
/// </summary>
|
||||
public bool Required { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Custom error message if element is not found (when Required=true)
|
||||
/// </summary>
|
||||
public string? ErrorMessage { get; set; }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
namespace BomLocalService.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Text patterns and regex configurations for parsing page content
|
||||
/// </summary>
|
||||
public class TextPatternsConfig
|
||||
{
|
||||
public string ResultsCountPattern { get; set; } = @"(\d+)\s+of\s+(\d+)";
|
||||
public string TimestampPattern { get; set; } = @"(?:[A-Za-z]+\s+)?\d{1,2}\s+[A-Za-z]{3},?\s+\d{1,2}:\d{2}\s+(?:am|pm)";
|
||||
public string ObservationTimePattern { get; set; } = @"Observations:\s*(\d+)\s*minutes?\s*ago";
|
||||
public string ForecastTimePattern { get; set; } = @"Forecast:\s*(\d+)\s*minutes?\s+ago";
|
||||
public string ForecastHourAgoPattern { get; set; } = @"Forecast:\s*an\s+hour\s+ago";
|
||||
public string WeatherStationPattern { get; set; } = @"at\s+([^,]+)\s+weather\s+station";
|
||||
public string DistancePattern { get; set; } = @"(\d+)\s*km\s+from";
|
||||
|
||||
public Dictionary<string, string> ExpectedTexts { get; set; } = new()
|
||||
{
|
||||
["PlayButtonLabel"] = "Play",
|
||||
["PauseButtonLabel"] = "Pause",
|
||||
["RadarLinkText"] = "Rain radar and weather map"
|
||||
};
|
||||
}
|
||||
|
||||
+56
@@ -1,5 +1,12 @@
|
||||
using BomLocalService.Services;
|
||||
using BomLocalService.Services.Interfaces;
|
||||
using BomLocalService.Services.Scraping;
|
||||
using BomLocalService.Services.Scraping.Steps.Navigation;
|
||||
using BomLocalService.Services.Scraping.Steps.Search;
|
||||
using BomLocalService.Services.Scraping.Steps.Map;
|
||||
using BomLocalService.Services.Scraping.Steps.Metadata;
|
||||
using BomLocalService.Services.Scraping.Steps.Capture;
|
||||
using BomLocalService.Services.Scraping.Workflows;
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
|
||||
@@ -8,6 +15,7 @@ var builder = WebApplication.CreateBuilder(args);
|
||||
builder.Services.AddControllersWithViews();
|
||||
builder.Services.AddEndpointsApiExplorer();
|
||||
builder.Services.AddOpenApi();
|
||||
builder.Services.AddHealthChecks();
|
||||
|
||||
// Configure CORS - MUST be added before other services
|
||||
var corsOrigins = builder.Configuration.GetValue<string>("Cors:AllowedOrigins", "*");
|
||||
@@ -82,6 +90,33 @@ builder.Services.AddSingleton<IDebugService, DebugService>();
|
||||
builder.Services.AddSingleton<ICacheService, CacheService>();
|
||||
builder.Services.AddSingleton<ITimeParsingService, TimeParsingService>();
|
||||
builder.Services.AddSingleton<IBrowserService, BrowserService>();
|
||||
builder.Services.AddSingleton<ISelectorService, SelectorService>();
|
||||
|
||||
// Register scraping step registry
|
||||
builder.Services.AddSingleton<IScrapingStepRegistry, ScrapingStepRegistry>();
|
||||
|
||||
// Register all scraping steps
|
||||
builder.Services.AddScoped<NavigateHomepageStep>();
|
||||
builder.Services.AddScoped<ClickSearchButtonStep>();
|
||||
builder.Services.AddScoped<FillSearchInputStep>();
|
||||
builder.Services.AddScoped<WaitForSearchResultsStep>();
|
||||
builder.Services.AddScoped<SelectSearchResultStep>();
|
||||
builder.Services.AddScoped<ClickRadarLinkStep>();
|
||||
builder.Services.AddScoped<WaitForMapReadyStep>();
|
||||
builder.Services.AddScoped<PauseRadarStep>();
|
||||
builder.Services.AddScoped<ResetToFirstFrameStep>();
|
||||
builder.Services.AddScoped<ExtractMetadataStep>();
|
||||
builder.Services.AddScoped<CalculateMapBoundsStep>();
|
||||
builder.Services.AddScoped<CaptureFramesStep>();
|
||||
|
||||
// Register workflows
|
||||
builder.Services.AddScoped<RadarScrapingWorkflow>();
|
||||
builder.Services.AddScoped<TemperatureMapWorkflow>();
|
||||
|
||||
// Register workflow factory
|
||||
builder.Services.AddSingleton<IWorkflowFactory, WorkflowFactory>();
|
||||
|
||||
// Register scraping service (depends on workflow factory)
|
||||
builder.Services.AddSingleton<IScrapingService, ScrapingService>();
|
||||
|
||||
// Register BOM Radar Service as singleton (orchestrator, depends on all above services)
|
||||
@@ -129,6 +164,27 @@ app.MapControllerRoute(
|
||||
// Map API controllers (with /api prefix)
|
||||
app.MapControllers();
|
||||
|
||||
// Map health check endpoint for Docker health monitoring
|
||||
app.MapHealthChecks("/api/health");
|
||||
|
||||
// Auto-register all scraping steps in the registry
|
||||
var stepRegistry = app.Services.GetRequiredService<IScrapingStepRegistry>();
|
||||
var stepTypes = typeof(IScrapingStep).Assembly.GetTypes()
|
||||
.Where(t => typeof(IScrapingStep).IsAssignableFrom(t) && !t.IsInterface && !t.IsAbstract && !t.IsGenericType);
|
||||
foreach (var stepType in stepTypes)
|
||||
{
|
||||
try
|
||||
{
|
||||
var step = (IScrapingStep)ActivatorUtilities.CreateInstance(app.Services, stepType);
|
||||
stepRegistry.RegisterStep(step);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
var logger = app.Services.GetRequiredService<ILogger<Program>>();
|
||||
logger.LogWarning(ex, "Failed to register step {StepType}", stepType.Name);
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup incomplete cache folders from previous crashes/restarts before starting services
|
||||
var cacheService = app.Services.GetRequiredService<ICacheService>();
|
||||
var deletedCount = cacheService.CleanupIncompleteCacheFolders();
|
||||
|
||||
@@ -41,10 +41,25 @@ Built on ASP.NET Core 9.0, the service uses a service-oriented architecture with
|
||||
- **BomRadarService**: Main orchestrator that coordinates cache operations, browser automation, and data retrieval
|
||||
- **CacheService**: Manages file-based storage of radar screenshots and metadata in organized directory structures
|
||||
- **BrowserService**: Handles Playwright browser automation for headless browser sessions
|
||||
- **ScrapingService**: Performs web scraping operations using the browser service to navigate and capture radar data from the BOM website
|
||||
- **ScrapingService**: Coordinates web scraping workflows (simplified orchestrator)
|
||||
- **SelectorService**: Finds page elements using configurable CSS selectors with fallback support
|
||||
- **TimeParsingService**: Parses and converts time formats from BOM data
|
||||
- **DebugService**: Provides debug functionality for troubleshooting
|
||||
|
||||
### Scraping Architecture
|
||||
The scraping system uses a **workflow-based architecture** with configurable steps:
|
||||
|
||||
- **Workflows**: Define fixed sequences of steps for different data types (e.g., `RadarScrapingWorkflow`, `TemperatureMapWorkflow`). Each workflow specifies its response type via generics (`IWorkflow<TResponse>`)
|
||||
- **Steps**: Individual, testable units that perform specific actions (navigation, search, map interaction, capture). Steps declare prerequisites and validate page state before execution
|
||||
- **Step Registry**: Manages and discovers available scraping steps
|
||||
- **Workflow Factory**: Creates typed workflow instances based on configuration
|
||||
|
||||
**Configuration-Driven Design:**
|
||||
- **Selectors**: All CSS selectors are configurable via `appsettings.json` with fallback options, allowing adaptation to website changes without code modifications
|
||||
- **JavaScript Templates**: JavaScript code for page evaluation is externalized in configuration, making it easy to update logic as the website evolves
|
||||
- **Text Patterns**: Regex patterns for parsing page content are configurable, enabling quick adjustments to parsing logic
|
||||
- **Workflow Steps**: Individual steps within workflows can be enabled/disabled via configuration, providing flexibility for testing and troubleshooting
|
||||
|
||||
### Background Services
|
||||
- **CacheManagementService**: Periodically checks cache validity for all cached locations and triggers updates when data expires
|
||||
- **CacheCleanupService**: Removes cache files older than the configured retention period
|
||||
@@ -163,11 +178,23 @@ If you want to build locally, keep the `build:` section in `docker-compose.yml`
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
**Mounting custom appsettings.json:**
|
||||
|
||||
To use a custom configuration file, add it to the volumes section in `docker-compose.yml`:
|
||||
```yaml
|
||||
services:
|
||||
bom-local-service:
|
||||
volumes:
|
||||
- ./cache:/app/cache
|
||||
- ./appsettings.json:/app/appsettings.json:ro # Custom config
|
||||
```
|
||||
|
||||
This will:
|
||||
- Pull/build the image as configured
|
||||
- Start the service on port 8082 (configurable via `HOST_PORT`)
|
||||
- Mount the `./cache` directory for persistent storage
|
||||
- Apply all environment variable configurations
|
||||
- Mount custom `appsettings.json` if specified
|
||||
- Apply all environment variable configurations (which override appsettings.json)
|
||||
|
||||
To view logs:
|
||||
```bash
|
||||
@@ -221,8 +248,50 @@ All configuration can be done via environment variables, which override the defa
|
||||
|
||||
| Variable | Description | Default | Example |
|
||||
|----------|-------------|---------|---------|
|
||||
| `SCREENSHOT__DYNAMICCONTENTWAITMS` | Milliseconds to wait for dynamic content to load | `2000` | `3000` |
|
||||
| `SCREENSHOT__TILERENDERWAITMS` | Milliseconds to wait for map tiles to render | `5000` | `7000` |
|
||||
| `SCREENSHOT__DYNAMICCONTENTWAITMS` | Milliseconds to wait for dynamic content to load | `1500` | `2000` |
|
||||
| `SCREENSHOT__TILERENDERWAITMS` | Milliseconds to wait for map tiles to render | `3000` | `5000` |
|
||||
| `SCREENSHOT__CROP__X` | X offset in pixels for screenshot cropping | `250` | `300` |
|
||||
| `SCREENSHOT__CROP__Y` | Y offset in pixels for screenshot cropping | `0` | `50` |
|
||||
| `SCREENSHOT__CROP__RIGHTOFFSET` | Right offset in pixels for screenshot cropping | `250` | `300` |
|
||||
| `SCREENSHOT__CROP__HEIGHT` | Height in pixels for screenshot cropping (null = full height) | `null` | `800` |
|
||||
|
||||
#### Scraping Configuration
|
||||
|
||||
The scraping system is highly configurable through `appsettings.json`. Most scraping settings (selectors, JavaScript templates, text patterns, workflow steps) are configured in `appsettings.json`, but can be overridden in Docker deployments.
|
||||
|
||||
**Option 1: Mount Custom appsettings.json (Recommended for Docker)**
|
||||
|
||||
Mount a custom `appsettings.json` file as a volume:
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
--name bom-local-service \
|
||||
-p 8082:8080 \
|
||||
-v $(pwd)/cache:/app/cache \
|
||||
-v $(pwd)/appsettings.json:/app/appsettings.json:ro \
|
||||
--shm-size=1gb \
|
||||
--ipc=host \
|
||||
ghcr.io/alexhopeoconnor/bom-local-service:latest
|
||||
```
|
||||
|
||||
Or in `docker-compose.yml`:
|
||||
```yaml
|
||||
services:
|
||||
bom-local-service:
|
||||
volumes:
|
||||
- ./cache:/app/cache
|
||||
- ./appsettings.json:/app/appsettings.json:ro # Add this line
|
||||
```
|
||||
|
||||
**Option 2: Environment Variables (Simple Overrides)**
|
||||
|
||||
For quick overrides of commonly needed settings:
|
||||
|
||||
| Variable | Description | Default | Example |
|
||||
|----------|-------------|---------|---------|
|
||||
| `SCRAPING__BASEURL` | Base URL for the BOM website | `https://www.bom.gov.au/` | `https://www.bom.gov.au/` |
|
||||
|
||||
**Note**: Complex configurations (selectors, JavaScript templates, text patterns, workflow steps) are best managed via a mounted `appsettings.json` file. See the [Configuration File](#configuration-file) section below for the complete structure.
|
||||
|
||||
#### Debug Configuration
|
||||
|
||||
@@ -306,6 +375,126 @@ Then run:
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
#### Custom appsettings.json (For Selector/Scraping Configuration)
|
||||
|
||||
If you need to customize selectors, JavaScript templates, or workflow steps:
|
||||
|
||||
1. **Copy the default appsettings.json** from the repository
|
||||
2. **Edit the sections you need** (e.g., `Scraping:Selectors`)
|
||||
3. **Mount it as a volume**:
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
--name bom-local-service \
|
||||
-p 8082:8080 \
|
||||
-v $(pwd)/cache:/app/cache \
|
||||
-v $(pwd)/appsettings.json:/app/appsettings.json:ro \
|
||||
--shm-size=1gb \
|
||||
--ipc=host \
|
||||
ghcr.io/alexhopeoconnor/bom-local-service:latest
|
||||
```
|
||||
|
||||
Or in `docker-compose.yml`:
|
||||
```yaml
|
||||
services:
|
||||
bom-local-service:
|
||||
volumes:
|
||||
- ./cache:/app/cache
|
||||
- ./appsettings.json:/app/appsettings.json:ro
|
||||
```
|
||||
|
||||
**Note**: Environment variables still override values in the mounted `appsettings.json`, so you can use env vars for simple overrides and the mounted file for complex configurations.
|
||||
|
||||
### Configuration File
|
||||
|
||||
For advanced scraping configuration, edit `appsettings.json` directly. When using Docker, you can mount a custom `appsettings.json` file as a volume (see [Scraping Configuration](#scraping-configuration) above).
|
||||
|
||||
The scraping system supports extensive configuration:
|
||||
|
||||
#### Scraping Selectors
|
||||
|
||||
All CSS selectors used to find page elements are configurable with fallback options:
|
||||
|
||||
```json
|
||||
{
|
||||
"Scraping": {
|
||||
"Selectors": {
|
||||
"SearchButton": {
|
||||
"Name": "Search Button",
|
||||
"Selectors": [
|
||||
"button[data-testid='searchLabel']",
|
||||
"button[aria-label='Search for a location']",
|
||||
"button.search-location__trigger-button"
|
||||
],
|
||||
"TimeoutMs": 10000,
|
||||
"Required": true,
|
||||
"ErrorMessage": "Could not find search button"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### JavaScript Templates
|
||||
|
||||
JavaScript code used for page evaluation is externalized and configurable:
|
||||
|
||||
```json
|
||||
{
|
||||
"Scraping": {
|
||||
"JavaScriptTemplates": {
|
||||
"WaitForSearchResults": "() => { /* template code */ }",
|
||||
"ExtractSearchResults": "() => { /* template code */ }"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Text Patterns
|
||||
|
||||
Regex patterns for parsing page content are configurable:
|
||||
|
||||
```json
|
||||
{
|
||||
"Scraping": {
|
||||
"TextPatterns": {
|
||||
"ResultsCountPattern": "(\\d+)\\s+of\\s+(\\d+)",
|
||||
"TimestampPattern": "(?:[A-Za-z]+\\s+)?\\d{1,2}\\s+[A-Za-z]{3},?\\s+\\d{1,2}:\\d{2}\\s+(?:am|pm)",
|
||||
"TimestampPattern": "(?:[A-Za-z]+\\s+)?\\d{1,2}\\s+[A-Za-z]{3},?\\s+\\d{1,2}:\\d{2}\\s+(?:am|pm)"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Workflow Steps
|
||||
|
||||
Individual workflow steps can be enabled/disabled and configured:
|
||||
|
||||
```json
|
||||
{
|
||||
"Scraping": {
|
||||
"Workflows": {
|
||||
"RadarScraping": {
|
||||
"Description": "Scrapes radar images for a location",
|
||||
"Steps": {
|
||||
"NavigateHomepage": { "Enabled": true },
|
||||
"ClickSearchButton": { "Enabled": true },
|
||||
"CaptureFrames": {
|
||||
"Enabled": true,
|
||||
"Parameters": {
|
||||
"FrameCount": 7,
|
||||
"WaitBetweenFrames": 5000
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Note**: Step order is fixed within workflows due to dependencies. Steps can be disabled but not reordered. See `appsettings.json` for the complete configuration structure.
|
||||
|
||||
## API Documentation
|
||||
|
||||
The service provides RESTful API endpoints for accessing radar data and managing the cache.
|
||||
@@ -711,7 +900,8 @@ The service uses a **metrics-based estimation system** to provide accurate estim
|
||||
|
||||
2. **Metrics Collection**: After each successful cache update, the service records:
|
||||
- Total duration of the update
|
||||
- Duration of each phase
|
||||
- Duration of each phase (Initializing, CapturingFrames, Saving)
|
||||
- Duration of each individual scraping step (NavigateHomepage, ClickSearchButton, etc.)
|
||||
- Frame-level progress during capture
|
||||
|
||||
3. **Estimation Strategy**:
|
||||
@@ -729,7 +919,7 @@ The service uses a **metrics-based estimation system** to provide accurate estim
|
||||
|
||||
**First Update (No Metrics)**:
|
||||
- Uses calculated estimate based on `Screenshot:DynamicContentWaitMs`, `Screenshot:TileRenderWaitMs`, and frame count
|
||||
- Example: ~120 seconds for 7 frames with default settings
|
||||
- Example: ~100 seconds for 7 frames with default settings (optimized wait times)
|
||||
|
||||
**Subsequent Updates (With Metrics)**:
|
||||
- Uses median duration from historical data
|
||||
@@ -998,6 +1188,25 @@ fi
|
||||
- **Check cache status**: Use `/api/cache/{suburb}/{state}/range` to verify cache exists
|
||||
- **Browser automation issues**: Check logs for Playwright errors
|
||||
|
||||
### Scraping Failures
|
||||
|
||||
If scraping fails (e.g., "Could not find element"), the BOM website structure may have changed:
|
||||
|
||||
- **Check debug screenshots**: Enable `DEBUG__ENABLED=true` to see what the browser sees at each step
|
||||
- **Update selectors**:
|
||||
- **Docker**: Mount a custom `appsettings.json` with updated selectors (see [Scraping Configuration](#scraping-configuration))
|
||||
- **Local**: Edit `appsettings.json` under `Scraping:Selectors` to add new CSS selectors as fallbacks
|
||||
- **Check step logs**: Each step logs its execution - look for which step failed
|
||||
- **Selector fallbacks**: The system tries multiple selectors in order, so add new selectors to the existing arrays
|
||||
- **Workflow steps**: Individual steps can be disabled via `Scraping:Workflows:RadarScraping:Steps:{StepName}:Enabled: false` if needed temporarily
|
||||
|
||||
**Quick Fix for Docker Users:**
|
||||
|
||||
1. Copy the default `appsettings.json` from the repository
|
||||
2. Edit the selectors that are failing
|
||||
3. Mount it as a volume: `-v $(pwd)/appsettings.json:/app/appsettings.json:ro`
|
||||
4. Restart the container
|
||||
|
||||
### Playwright Resource Usage
|
||||
|
||||
Playwright browsers (Chromium) can consume significant CPU and memory:
|
||||
@@ -1031,6 +1240,25 @@ docker run --cpus="1.0" ...
|
||||
- **Increase cleanup frequency**: Lower `CACHECLEANUP__INTERVALHOURS` to clean up more often
|
||||
- **Limit locations**: The service automatically manages all cached locations; reduce the number of locations being cached to lower resource usage
|
||||
|
||||
### Performance Monitoring
|
||||
|
||||
The service logs detailed performance metrics for each scraping workflow:
|
||||
|
||||
**Step-Level Timing**:
|
||||
- Each step logs its duration and compares it to historical averages
|
||||
- Example: `Step WaitForMapReady completed in 45.30s (avg: 43.76s)`
|
||||
- Steps that are >50% slower than average trigger warnings: `⚠️ Step WaitForMapReady took significantly longer than average: 75.45s (avg: 50.30s, +25.15s, +50.0% slower)`
|
||||
|
||||
**Workflow-Level Timing**:
|
||||
- Complete workflow duration is logged with step breakdown
|
||||
- Example: `Workflow RadarScraping completed in 144.40s. Step breakdown: NavigateHomepage=4.96s, ClickSearchButton=2.70s, ...`
|
||||
- Workflows that are >30% slower than average trigger warnings: `⚠️ Workflow RadarScraping took significantly longer than average: 189.45s (avg: 145.67s, +43.78s, +30.0% slower)`
|
||||
|
||||
**Metrics Storage**:
|
||||
- Step and phase durations are stored in memory (last 20 samples)
|
||||
- Used for performance estimation and identifying bottlenecks
|
||||
- Metrics improve over time as more updates complete
|
||||
|
||||
## Development
|
||||
|
||||
### Building Locally
|
||||
@@ -1049,7 +1277,45 @@ Enable debug mode to save intermediate screenshots during data capture:
|
||||
docker run -e DEBUG__ENABLED=true -e DEBUG__WAITMS=5000 ...
|
||||
```
|
||||
|
||||
Debug screenshots are saved in `{CACHEDIRECTORY}/debug/`.
|
||||
Debug screenshots are saved in `{CACHEDIRECTORY}/debug/`. Each scraping step saves a screenshot, HTML snapshot, and logs, making it easy to diagnose issues.
|
||||
|
||||
### Extending the Scraping System
|
||||
|
||||
The workflow-based architecture makes it easy to extend the scraping system:
|
||||
|
||||
**Adding a New Workflow**:
|
||||
|
||||
1. Create a new workflow class in `Services/Scraping/Workflows/` implementing `IWorkflow<TResponse>` where `TResponse` is your response type
|
||||
2. Define the step sequence (can reuse existing steps)
|
||||
3. Register the workflow in `WorkflowFactory`
|
||||
4. Add workflow configuration to `appsettings.json`
|
||||
|
||||
**Adding a New Step**:
|
||||
|
||||
1. Create a step class inheriting from `BaseScrapingStep`
|
||||
2. Implement `Name`, `Prerequisites`, `CanExecute`, and `ExecuteAsync`
|
||||
3. The step will be auto-registered on startup
|
||||
4. Add the step to a workflow's `StepNames` array
|
||||
|
||||
**Updating Selectors**:
|
||||
|
||||
1. Edit `appsettings.json` under `Scraping:Selectors`
|
||||
2. Add new CSS selectors to the `Selectors` array (tried in order)
|
||||
3. Adjust `TimeoutMs` if needed
|
||||
4. No code changes required
|
||||
|
||||
**Updating JavaScript Templates**:
|
||||
|
||||
1. Edit `appsettings.json` under `Scraping:JavaScriptTemplates`
|
||||
2. Update the template code as needed
|
||||
3. No code changes required
|
||||
|
||||
**Updating Text Patterns**:
|
||||
|
||||
1. Edit `appsettings.json` under `Scraping:TextPatterns`
|
||||
2. Update regex patterns as needed (e.g., `TimestampPattern`)
|
||||
3. The `TimestampPattern` supports parsing timestamps like "Wednesday 17 Dec, 11:05 pm" when the BOM website changes format
|
||||
4. No code changes required
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ public class CacheService : ICacheService
|
||||
private readonly ConcurrentDictionary<string, (DateTime startTime, CacheUpdatePhase phase, int? currentFrame, int? totalFrames)> _updateProgress = new();
|
||||
private readonly ConcurrentQueue<double> _recentTotalDurations = new(); // Overall durations in seconds
|
||||
private readonly ConcurrentDictionary<CacheUpdatePhase, ConcurrentQueue<double>> _phaseDurations = new(); // Phase -> durations
|
||||
private readonly ConcurrentDictionary<string, ConcurrentQueue<double>> _stepDurations = new(); // Step name -> durations
|
||||
private readonly object _metricsLock = new();
|
||||
private const int MaxSamples = 20;
|
||||
|
||||
@@ -589,7 +590,7 @@ public class CacheService : ICacheService
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogDebug("Cache update completed in {Duration:F1} seconds for {Location}", totalDuration, locationKey);
|
||||
_logger.LogInformation("Cache update completed in {Duration:F1} seconds for {Location}", totalDuration, locationKey);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -665,7 +666,7 @@ public class CacheService : ICacheService
|
||||
/// <summary>
|
||||
/// Gets the average total duration of cache updates from recent metrics.
|
||||
/// </summary>
|
||||
private double GetAverageTotalDuration()
|
||||
public double GetAverageTotalDuration()
|
||||
{
|
||||
lock (_metricsLock)
|
||||
{
|
||||
@@ -707,6 +708,59 @@ public class CacheService : ICacheService
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records step completion timing for metrics tracking.
|
||||
/// </summary>
|
||||
public void RecordStepCompletion(string stepName, double durationSeconds)
|
||||
{
|
||||
lock (_metricsLock)
|
||||
{
|
||||
var durations = _stepDurations.GetOrAdd(stepName, _ => new ConcurrentQueue<double>());
|
||||
durations.Enqueue(durationSeconds);
|
||||
|
||||
while (durations.Count > MaxSamples)
|
||||
{
|
||||
durations.TryDequeue(out _);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the average duration for a specific step from historical data.
|
||||
/// </summary>
|
||||
public double GetAverageStepDuration(string stepName)
|
||||
{
|
||||
lock (_metricsLock)
|
||||
{
|
||||
if (!_stepDurations.TryGetValue(stepName, out var durations) || durations.Count == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
var durationsArray = durations.ToArray();
|
||||
return durationsArray.Average();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets step performance metrics for debugging/logging.
|
||||
/// </summary>
|
||||
public Dictionary<string, double> GetStepMetrics()
|
||||
{
|
||||
lock (_metricsLock)
|
||||
{
|
||||
var metrics = new Dictionary<string, double>();
|
||||
foreach (var kvp in _stepDurations)
|
||||
{
|
||||
if (kvp.Value.Count > 0)
|
||||
{
|
||||
var durations = kvp.Value.ToArray();
|
||||
metrics[kvp.Key] = durations.Average();
|
||||
}
|
||||
}
|
||||
return metrics;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the locationKey from a cacheFolderPath by parsing the folder name.
|
||||
/// </summary>
|
||||
|
||||
@@ -90,18 +90,22 @@ namespace BomLocalService.Services;
|
||||
await File.WriteAllTextAsync(htmlPath, htmlContent, cancellationToken);
|
||||
|
||||
// Save console messages if provided
|
||||
// Create a snapshot to avoid collection modification during enumeration
|
||||
if (consoleMessages != null && consoleMessages.Count > 0)
|
||||
{
|
||||
var consolePath = Path.Combine(stepFolder, "console.log");
|
||||
var consoleText = string.Join("\n", consoleMessages.Select(msg => $"[{msg.timestamp:HH:mm:ss.fff}] [{msg.type}] {msg.text}"));
|
||||
var consoleSnapshot = consoleMessages.ToList(); // Create snapshot
|
||||
var consoleText = string.Join("\n", consoleSnapshot.Select(msg => $"[{msg.timestamp:HH:mm:ss.fff}] [{msg.type}] {msg.text}"));
|
||||
await File.WriteAllTextAsync(consolePath, consoleText, cancellationToken);
|
||||
}
|
||||
|
||||
// Save network requests summary if provided
|
||||
// Create a snapshot to avoid collection modification during enumeration
|
||||
if (networkRequests != null && networkRequests.Count > 0)
|
||||
{
|
||||
var networkPath = Path.Combine(stepFolder, "network.log");
|
||||
var networkText = string.Join("\n", networkRequests.Select(req =>
|
||||
var networkSnapshot = networkRequests.ToList(); // Create snapshot
|
||||
var networkText = string.Join("\n", networkSnapshot.Select(req =>
|
||||
$"[{req.timestamp:HH:mm:ss.fff}] {req.method} {req.url} -> {req.status?.ToString() ?? "pending"} ({req.resourceType})"
|
||||
));
|
||||
await File.WriteAllTextAsync(networkPath, networkText, cancellationToken);
|
||||
@@ -148,18 +152,22 @@ namespace BomLocalService.Services;
|
||||
}
|
||||
|
||||
// Save console messages if provided
|
||||
// Create a snapshot to avoid collection modification during enumeration
|
||||
if (consoleMessages != null && consoleMessages.Count > 0)
|
||||
{
|
||||
var consolePath = Path.Combine(errorFolder, "console.log");
|
||||
var consoleText = string.Join("\n", consoleMessages.Select(msg => $"[{msg.timestamp:HH:mm:ss.fff}] [{msg.type}] {msg.text}"));
|
||||
var consoleSnapshot = consoleMessages.ToList(); // Create snapshot
|
||||
var consoleText = string.Join("\n", consoleSnapshot.Select(msg => $"[{msg.timestamp:HH:mm:ss.fff}] [{msg.type}] {msg.text}"));
|
||||
await File.WriteAllTextAsync(consolePath, consoleText, cancellationToken);
|
||||
}
|
||||
|
||||
// Save network requests summary if provided
|
||||
// Create a snapshot to avoid collection modification during enumeration
|
||||
if (networkRequests != null && networkRequests.Count > 0)
|
||||
{
|
||||
var networkPath = Path.Combine(errorFolder, "network.log");
|
||||
var networkText = string.Join("\n", networkRequests.Select(req =>
|
||||
var networkSnapshot = networkRequests.ToList(); // Create snapshot
|
||||
var networkText = string.Join("\n", networkSnapshot.Select(req =>
|
||||
$"[{req.timestamp:HH:mm:ss.fff}] {req.method} {req.url} -> {req.status?.ToString() ?? "pending"} ({req.resourceType})"
|
||||
));
|
||||
await File.WriteAllTextAsync(networkPath, networkText, cancellationToken);
|
||||
|
||||
@@ -213,5 +213,32 @@ public interface ICacheService
|
||||
/// <param name="locationKey">The location key (suburb_state)</param>
|
||||
/// <returns>Estimated remaining seconds, or 0 if not updating or no metrics</returns>
|
||||
int GetEstimatedRemainingSeconds(string locationKey);
|
||||
|
||||
/// <summary>
|
||||
/// Records step completion timing for metrics tracking.
|
||||
/// </summary>
|
||||
/// <param name="stepName">The name of the step</param>
|
||||
/// <param name="durationSeconds">The duration of the step in seconds</param>
|
||||
void RecordStepCompletion(string stepName, double durationSeconds);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the average duration for a specific step from historical data.
|
||||
/// </summary>
|
||||
/// <param name="stepName">The name of the step</param>
|
||||
/// <returns>Average duration in seconds, or 0 if no data available</returns>
|
||||
double GetAverageStepDuration(string stepName);
|
||||
|
||||
/// <summary>
|
||||
/// Gets step performance metrics for debugging/logging.
|
||||
/// </summary>
|
||||
/// <returns>Dictionary mapping step names to their average durations</returns>
|
||||
Dictionary<string, double> GetStepMetrics();
|
||||
|
||||
/// <summary>
|
||||
/// Gets the average total duration of cache updates from recent metrics.
|
||||
/// Uses median for robustness against outliers.
|
||||
/// </summary>
|
||||
/// <returns>Average total duration in seconds, or 0 if no data available</returns>
|
||||
double GetAverageTotalDuration();
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
using BomLocalService.Models;
|
||||
using Microsoft.Playwright;
|
||||
|
||||
namespace BomLocalService.Services.Interfaces;
|
||||
|
||||
/// <summary>
|
||||
/// Service for finding page elements using configured selectors
|
||||
/// </summary>
|
||||
public interface ISelectorService
|
||||
{
|
||||
/// <summary>
|
||||
/// Finds an element using the configured selectors, trying each in order until one is found
|
||||
/// </summary>
|
||||
Task<ILocator?> FindElementAsync(IPage page, SelectorConfig config, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets a locator for the first selector (assumes element exists)
|
||||
/// </summary>
|
||||
ILocator GetLocator(IPage page, SelectorConfig config);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
namespace BomLocalService.Services.Scraping;
|
||||
|
||||
/// <summary>
|
||||
/// Interface for a single scraping step
|
||||
/// </summary>
|
||||
public interface IScrapingStep
|
||||
{
|
||||
/// <summary>
|
||||
/// Unique name of the step
|
||||
/// </summary>
|
||||
string Name { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Names of steps that must complete before this step can execute
|
||||
/// </summary>
|
||||
string[] Prerequisites { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Checks if the step can execute in the current page state
|
||||
/// </summary>
|
||||
bool CanExecute(ScrapingContext context);
|
||||
|
||||
/// <summary>
|
||||
/// Executes the step
|
||||
/// </summary>
|
||||
Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
namespace BomLocalService.Services.Scraping;
|
||||
|
||||
/// <summary>
|
||||
/// Registry for managing scraping steps
|
||||
/// </summary>
|
||||
public interface IScrapingStepRegistry
|
||||
{
|
||||
void RegisterStep(IScrapingStep step);
|
||||
IScrapingStep? GetStep(string name);
|
||||
IEnumerable<IScrapingStep> GetAllSteps();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
namespace BomLocalService.Services.Scraping;
|
||||
|
||||
/// <summary>
|
||||
/// Interface for a scraping workflow
|
||||
/// </summary>
|
||||
/// <typeparam name="TResponse">The response type returned by this workflow</typeparam>
|
||||
public interface IWorkflow<TResponse>
|
||||
{
|
||||
string Name { get; }
|
||||
string Description { get; }
|
||||
string[] StepNames { get; } // Fixed order - cannot be changed
|
||||
Task<TResponse> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
namespace BomLocalService.Services.Scraping;
|
||||
|
||||
/// <summary>
|
||||
/// Factory for creating workflows
|
||||
/// </summary>
|
||||
public interface IWorkflowFactory
|
||||
{
|
||||
IWorkflow<TResponse> GetWorkflow<TResponse>(string name);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
namespace BomLocalService.Services.Scraping;
|
||||
|
||||
/// <summary>
|
||||
/// Represents the current state of the page during scraping
|
||||
/// </summary>
|
||||
public enum PageState
|
||||
{
|
||||
Initial,
|
||||
HomepageLoaded,
|
||||
SearchModalOpen,
|
||||
SearchResultsVisible,
|
||||
ForecastPageLoaded,
|
||||
RadarPageLoaded,
|
||||
MapReady,
|
||||
SlideshowPaused,
|
||||
Frame0Selected
|
||||
}
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
using BomLocalService.Models;
|
||||
using Microsoft.Playwright;
|
||||
|
||||
namespace BomLocalService.Services.Scraping;
|
||||
|
||||
/// <summary>
|
||||
/// Context shared between scraping steps
|
||||
/// </summary>
|
||||
public class ScrapingContext
|
||||
{
|
||||
public IPage Page { get; set; } = null!;
|
||||
public string Suburb { get; set; } = string.Empty;
|
||||
public string State { get; set; } = string.Empty;
|
||||
public string CacheFolderPath { get; set; } = string.Empty;
|
||||
public string DebugFolder { get; set; } = string.Empty;
|
||||
public List<(string type, string text, DateTime timestamp)> ConsoleMessages { get; set; } = new();
|
||||
public List<(string method, string url, int? status, string resourceType, DateTime timestamp)> NetworkRequests { get; set; } = new();
|
||||
|
||||
// Page state tracking
|
||||
public PageState CurrentState { get; set; } = PageState.Initial;
|
||||
public HashSet<string> CompletedSteps { get; set; } = new();
|
||||
|
||||
// Shared state between steps
|
||||
public ILocator? SearchButton { get; set; }
|
||||
public ILocator? SearchInput { get; set; }
|
||||
public List<(string name, string desc, string fullText)>? SearchResults { get; set; }
|
||||
public int? SelectedResultIndex { get; set; }
|
||||
public ILocator? RadarLink { get; set; }
|
||||
public ILocator? MapContainer { get; set; }
|
||||
public Clip? MapBoundingBox { get; set; }
|
||||
public LastUpdatedInfo? LastUpdatedInfo { get; set; }
|
||||
public List<(int index, int minutesAgo)>? FrameInfo { get; set; }
|
||||
public List<RadarFrame> Frames { get; set; } = new();
|
||||
|
||||
// State validation helpers
|
||||
public bool IsHomepageLoaded => CurrentState >= PageState.HomepageLoaded;
|
||||
public bool IsSearchModalOpen => CurrentState >= PageState.SearchModalOpen;
|
||||
public bool IsForecastPageLoaded => CurrentState >= PageState.ForecastPageLoaded;
|
||||
public bool IsRadarPageLoaded => CurrentState >= PageState.RadarPageLoaded;
|
||||
public bool IsMapReady => CurrentState >= PageState.MapReady;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
namespace BomLocalService.Services.Scraping;
|
||||
|
||||
public class ScrapingStepRegistry : IScrapingStepRegistry
|
||||
{
|
||||
private readonly Dictionary<string, IScrapingStep> _steps = new();
|
||||
private readonly ILogger<ScrapingStepRegistry> _logger;
|
||||
|
||||
public ScrapingStepRegistry(ILogger<ScrapingStepRegistry> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public void RegisterStep(IScrapingStep step)
|
||||
{
|
||||
if (_steps.ContainsKey(step.Name))
|
||||
{
|
||||
_logger.LogWarning("Step {Name} is already registered, overwriting", step.Name);
|
||||
}
|
||||
_steps[step.Name] = step;
|
||||
_logger.LogDebug("Registered step: {Name}", step.Name);
|
||||
}
|
||||
|
||||
public IScrapingStep? GetStep(string name)
|
||||
{
|
||||
return _steps.TryGetValue(name, out var step) ? step : null;
|
||||
}
|
||||
|
||||
public IEnumerable<IScrapingStep> GetAllSteps()
|
||||
{
|
||||
return _steps.Values;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
namespace BomLocalService.Services.Scraping;
|
||||
|
||||
/// <summary>
|
||||
/// Result of executing a scraping step
|
||||
/// </summary>
|
||||
public class ScrapingStepResult
|
||||
{
|
||||
public bool Success { get; set; }
|
||||
public string? ErrorMessage { get; set; }
|
||||
public Dictionary<string, object>? Data { get; set; }
|
||||
|
||||
public static ScrapingStepResult Successful() => new() { Success = true };
|
||||
public static ScrapingStepResult Failed(string errorMessage) => new() { Success = false, ErrorMessage = errorMessage };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
using BomLocalService.Models;
|
||||
using BomLocalService.Services.Interfaces;
|
||||
using Microsoft.Playwright;
|
||||
|
||||
namespace BomLocalService.Services.Scraping.Steps;
|
||||
|
||||
/// <summary>
|
||||
/// Base class for scraping steps with common functionality
|
||||
/// </summary>
|
||||
public abstract class BaseScrapingStep : IScrapingStep
|
||||
{
|
||||
protected readonly ILogger Logger;
|
||||
protected readonly ISelectorService SelectorService;
|
||||
protected readonly IDebugService DebugService;
|
||||
protected readonly IConfiguration Configuration;
|
||||
protected readonly ScrapingSelectorsConfig Selectors;
|
||||
protected readonly JavaScriptTemplatesConfig JavaScriptTemplates;
|
||||
protected readonly TextPatternsConfig TextPatterns;
|
||||
|
||||
protected BaseScrapingStep(
|
||||
ILogger logger,
|
||||
ISelectorService selectorService,
|
||||
IDebugService debugService,
|
||||
IConfiguration configuration)
|
||||
{
|
||||
Logger = logger;
|
||||
SelectorService = selectorService;
|
||||
DebugService = debugService;
|
||||
Configuration = configuration;
|
||||
|
||||
Selectors = configuration.GetSection("Scraping:Selectors").Get<ScrapingSelectorsConfig>() ?? new();
|
||||
JavaScriptTemplates = configuration.GetSection("Scraping:JavaScriptTemplates").Get<JavaScriptTemplatesConfig>() ?? new();
|
||||
TextPatterns = configuration.GetSection("Scraping:TextPatterns").Get<TextPatternsConfig>() ?? new();
|
||||
}
|
||||
|
||||
public abstract string Name { get; }
|
||||
public abstract string[] Prerequisites { get; }
|
||||
public abstract bool CanExecute(ScrapingContext context);
|
||||
public abstract Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken);
|
||||
|
||||
protected async Task SaveDebugAsync(ScrapingContext context, int stepNumber, string stepName, CancellationToken cancellationToken)
|
||||
{
|
||||
await DebugService.SaveStepDebugAsync(
|
||||
context.DebugFolder,
|
||||
stepNumber,
|
||||
stepName,
|
||||
context.Page,
|
||||
context.ConsoleMessages,
|
||||
context.NetworkRequests,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
protected async Task SaveErrorDebugAsync(ScrapingContext context, string errorMessage, CancellationToken cancellationToken)
|
||||
{
|
||||
await DebugService.SaveErrorDebugAsync(
|
||||
context.DebugFolder,
|
||||
errorMessage,
|
||||
context.Page,
|
||||
context.ConsoleMessages,
|
||||
context.NetworkRequests,
|
||||
cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,489 @@
|
||||
using BomLocalService.Models;
|
||||
using BomLocalService.Services.Interfaces;
|
||||
using BomLocalService.Services.Scraping;
|
||||
using BomLocalService.Utilities;
|
||||
using Microsoft.Playwright;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace BomLocalService.Services.Scraping.Steps.Capture;
|
||||
|
||||
public class CaptureFramesStep : BaseScrapingStep
|
||||
{
|
||||
private readonly ICacheService _cacheService;
|
||||
private readonly int _tileRenderWaitMs;
|
||||
private readonly ScreenshotCropConfig _cropConfig;
|
||||
|
||||
public override string Name => "CaptureFrames";
|
||||
public override string[] Prerequisites => new[] { "ExtractMetadata", "CalculateMapBounds" };
|
||||
|
||||
public CaptureFramesStep(
|
||||
ILogger<CaptureFramesStep> logger,
|
||||
ISelectorService selectorService,
|
||||
IDebugService debugService,
|
||||
IConfiguration configuration,
|
||||
ICacheService cacheService)
|
||||
: base(logger, selectorService, debugService, configuration)
|
||||
{
|
||||
_cacheService = cacheService;
|
||||
_tileRenderWaitMs = configuration.GetValue<int>("Screenshot:TileRenderWaitMs", 5000);
|
||||
|
||||
var cropSection = configuration.GetSection("Screenshot:Crop");
|
||||
_cropConfig = new ScreenshotCropConfig
|
||||
{
|
||||
X = cropSection.GetValue<int>("X", 0),
|
||||
Y = cropSection.GetValue<int>("Y", 0),
|
||||
RightOffset = cropSection.GetValue<int>("RightOffset", 0),
|
||||
Height = cropSection.GetValue<int?>("Height")
|
||||
};
|
||||
}
|
||||
|
||||
public override bool CanExecute(ScrapingContext context)
|
||||
{
|
||||
return context.MapBoundingBox != null && context.MapContainer != null;
|
||||
}
|
||||
|
||||
public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(context.CacheFolderPath);
|
||||
Logger.LogInformation("Step {Step}: Using cache folder: {Path}", Name, context.CacheFolderPath);
|
||||
|
||||
var frameCount = CacheHelper.GetFrameCountForDataType(Configuration, CachedDataType.Radar);
|
||||
|
||||
_cacheService.RecordUpdateProgressByFolder(context.CacheFolderPath, CacheUpdatePhase.CapturingFrames, 0, frameCount);
|
||||
|
||||
var frames = new List<RadarFrame>();
|
||||
var stepForwardButton = SelectorService.GetLocator(context.Page, Selectors.StepForwardButton);
|
||||
int? previousMinutesAgo = null;
|
||||
|
||||
for (int frameIndex = 0; frameIndex < frameCount; frameIndex++)
|
||||
{
|
||||
Logger.LogInformation("Step {Step}: Capturing frame {FrameIndex} of {FrameCount}", Name, frameIndex, frameCount);
|
||||
|
||||
await context.Page.WaitForTimeoutAsync(_tileRenderWaitMs);
|
||||
|
||||
// Small wait to ensure display label is stable after frame change/selection
|
||||
// This is especially important for frame 0 which was just selected in ResetToFirstFrame
|
||||
await context.Page.WaitForTimeoutAsync(300);
|
||||
|
||||
// Try extracting with a retry in case the label is still updating
|
||||
var minutesAgo = await ExtractMinutesAgoFromDisplayAsync(context.Page);
|
||||
if (minutesAgo == null)
|
||||
{
|
||||
// Retry once after a short wait in case label was updating
|
||||
await context.Page.WaitForTimeoutAsync(200);
|
||||
minutesAgo = await ExtractMinutesAgoFromDisplayAsync(context.Page);
|
||||
}
|
||||
if (minutesAgo == null && context.FrameInfo != null && frameIndex < context.FrameInfo.Count)
|
||||
{
|
||||
var (_, defaultMinutesAgo) = context.FrameInfo[frameIndex];
|
||||
minutesAgo = defaultMinutesAgo;
|
||||
Logger.LogWarning("Step {Step}: Failed to extract minutes from display label for frame {FrameIndex}, using default: {MinutesAgo}", Name, frameIndex, minutesAgo);
|
||||
}
|
||||
|
||||
if (frameIndex > 0 && previousMinutesAgo.HasValue && minutesAgo == previousMinutesAgo.Value)
|
||||
{
|
||||
Logger.LogWarning("Step {Step}: Frame {FrameIndex} has same minutesAgo ({MinutesAgo}) as previous frame. Waiting for display to update...", Name, frameIndex, minutesAgo);
|
||||
await WaitForDisplayLabelToChangeAsync(context.Page, previousMinutesAgo.Value);
|
||||
minutesAgo = await ExtractMinutesAgoFromDisplayAsync(context.Page);
|
||||
if (minutesAgo == null || minutesAgo == previousMinutesAgo.Value)
|
||||
{
|
||||
if (context.FrameInfo != null && frameIndex < context.FrameInfo.Count)
|
||||
{
|
||||
var (_, defaultMinutesAgo) = context.FrameInfo[frameIndex];
|
||||
minutesAgo = defaultMinutesAgo;
|
||||
Logger.LogWarning("Step {Step}: Display label did not update for frame {FrameIndex}, using calculated default: {MinutesAgo}", Name, frameIndex, minutesAgo);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var radarFolder = FilePathHelper.GetDataTypeFolderPath(context.CacheFolderPath, CachedDataType.Radar);
|
||||
if (!Directory.Exists(radarFolder))
|
||||
{
|
||||
Directory.CreateDirectory(radarFolder);
|
||||
}
|
||||
|
||||
var framePath = FilePathHelper.GetFrameFilePath(context.CacheFolderPath, CachedDataType.Radar, frameIndex);
|
||||
await CaptureMapScreenshotAsync(context.Page, context.MapContainer!, framePath, context.MapBoundingBox!);
|
||||
|
||||
frames.Add(new RadarFrame
|
||||
{
|
||||
FrameIndex = frameIndex,
|
||||
ImagePath = framePath,
|
||||
MinutesAgo = minutesAgo ?? 0
|
||||
});
|
||||
|
||||
previousMinutesAgo = minutesAgo;
|
||||
|
||||
Logger.LogInformation("Step {Step}: Frame {FrameIndex} saved: {Path} ({MinutesAgo} minutes ago)",
|
||||
Name, frameIndex, framePath, minutesAgo ?? 0);
|
||||
|
||||
_cacheService.RecordUpdateProgressByFolder(context.CacheFolderPath, CacheUpdatePhase.CapturingFrames, frameIndex + 1, frameCount);
|
||||
|
||||
await SaveDebugAsync(context, 15 + frameIndex, $"frame_{frameIndex}_captured", cancellationToken);
|
||||
|
||||
if (frameIndex < frameCount - 1)
|
||||
{
|
||||
await DismissModalOverlaysAsync(context.Page);
|
||||
|
||||
var currentMinutesAgo = await ExtractMinutesAgoFromDisplayAsync(context.Page);
|
||||
|
||||
await stepForwardButton.ClickAsync(new LocatorClickOptions { Force = true });
|
||||
|
||||
if (currentMinutesAgo.HasValue)
|
||||
{
|
||||
await WaitForDisplayLabelToChangeAsync(context.Page, currentMinutesAgo.Value);
|
||||
}
|
||||
else
|
||||
{
|
||||
await context.Page.WaitForTimeoutAsync(500);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Logger.LogInformation("Step {Step}: All {FrameCount} frames captured successfully", Name, frameCount);
|
||||
|
||||
_cacheService.RecordUpdateProgressByFolder(context.CacheFolderPath, CacheUpdatePhase.Saving);
|
||||
|
||||
if (context.LastUpdatedInfo != null)
|
||||
{
|
||||
await _cacheService.SaveMetadataAsync(context.CacheFolderPath, context.LastUpdatedInfo, cancellationToken);
|
||||
}
|
||||
await _cacheService.SaveFramesMetadataAsync(context.CacheFolderPath, CachedDataType.Radar, frames, cancellationToken);
|
||||
|
||||
context.Frames = frames;
|
||||
|
||||
return ScrapingStepResult.Successful();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogError(ex, "Step {Step} failed", Name);
|
||||
await SaveErrorDebugAsync(context, $"Failed to capture frames: {ex.Message}", cancellationToken);
|
||||
return ScrapingStepResult.Failed($"Failed to capture frames: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<int?> ExtractMinutesAgoFromDisplayAsync(IPage page)
|
||||
{
|
||||
try
|
||||
{
|
||||
var timeLabelLocator = SelectorService.GetLocator(page, Selectors.TimeDisplayLabel);
|
||||
var timeLabel = await timeLabelLocator.TextContentAsync();
|
||||
if (string.IsNullOrEmpty(timeLabel))
|
||||
{
|
||||
Logger.LogDebug("Time display label is empty");
|
||||
return null;
|
||||
}
|
||||
|
||||
var trimmedLabel = timeLabel.Trim();
|
||||
Logger.LogInformation("Extracting minutes from display label: '{Label}'", trimmedLabel);
|
||||
|
||||
// Parse timestamp format (current BOM website format): "Wednesday 17 Dec, 11:05 pm" or "17 Dec, 11:05 pm"
|
||||
Logger.LogInformation("Trying timestamp pattern: '{Pattern}'", TextPatterns.TimestampPattern);
|
||||
var timestampMatch = Regex.Match(trimmedLabel, TextPatterns.TimestampPattern, RegexOptions.IgnoreCase);
|
||||
if (timestampMatch.Success)
|
||||
{
|
||||
var timestampStr = timestampMatch.Groups[0].Value;
|
||||
Logger.LogInformation("Matched timestamp pattern: '{Timestamp}' from label: '{Label}'", timestampStr, trimmedLabel);
|
||||
if (TryParseTimestamp(timestampStr, out var timestamp))
|
||||
{
|
||||
var minutesAgo = (int)(DateTime.UtcNow - timestamp).TotalMinutes;
|
||||
Logger.LogInformation("Parsed timestamp: {Timestamp} UTC, calculated minutes ago: {Minutes}", timestamp, minutesAgo);
|
||||
if (minutesAgo >= 0 && minutesAgo <= 120) // Reasonable range: 0-2 hours
|
||||
{
|
||||
Logger.LogInformation("Successfully calculated minutes ago from timestamp: {Minutes}", minutesAgo);
|
||||
return minutesAgo;
|
||||
}
|
||||
else
|
||||
{
|
||||
Logger.LogWarning("Calculated minutes ago ({Minutes}) outside reasonable range (0-120)", minutesAgo);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Logger.LogWarning("Failed to parse timestamp string: '{Timestamp}'", timestampStr);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Logger.LogWarning("Display label did not match timestamp pattern. Label: '{Label}', Pattern: '{Pattern}'", trimmedLabel, TextPatterns.TimestampPattern);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogDebug(ex, "Failed to extract minutes from display label");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private bool TryParseTimestamp(string timestampStr, out DateTime timestamp)
|
||||
{
|
||||
timestamp = DateTime.MinValue;
|
||||
|
||||
try
|
||||
{
|
||||
// Try common Australian date formats
|
||||
// Format: "Wednesday 17 Dec, 11:05 pm" or "17 Dec, 11:05 pm"
|
||||
var formats = new[]
|
||||
{
|
||||
"dddd d MMM, h:mm tt", // Wednesday 17 Dec, 11:05 pm
|
||||
"d MMM, h:mm tt", // 17 Dec, 11:05 pm
|
||||
"dddd d MMM, hh:mm tt", // Wednesday 17 Dec, 11:05 pm (with leading zero)
|
||||
"d MMM, hh:mm tt", // 17 Dec, 11:05 pm (with leading zero)
|
||||
"dddd dd MMM, h:mm tt", // Wednesday 17 Dec, 11:05 pm (with leading zero day)
|
||||
"dd MMM, h:mm tt" // 17 Dec, 11:05 pm (with leading zero day)
|
||||
};
|
||||
|
||||
var culture = new System.Globalization.CultureInfo("en-AU");
|
||||
|
||||
foreach (var format in formats)
|
||||
{
|
||||
if (DateTime.TryParseExact(timestampStr, format, culture,
|
||||
System.Globalization.DateTimeStyles.AssumeLocal, out timestamp))
|
||||
{
|
||||
// If year is not specified, assume current year
|
||||
if (timestamp.Year == 1)
|
||||
{
|
||||
timestamp = new DateTime(DateTime.Now.Year, timestamp.Month, timestamp.Day,
|
||||
timestamp.Hour, timestamp.Minute, timestamp.Second);
|
||||
}
|
||||
|
||||
// If the parsed time is in the future (likely same day next year), adjust
|
||||
if (timestamp > DateTime.Now && timestamp < DateTime.Now.AddDays(1))
|
||||
{
|
||||
// Already correct
|
||||
}
|
||||
else if (timestamp > DateTime.Now)
|
||||
{
|
||||
// Likely parsed as next year, adjust to this year
|
||||
timestamp = timestamp.AddYears(-1);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task WaitForDisplayLabelToChangeAsync(IPage page, int currentMinutesAgo, int maxWaitMs = 5000)
|
||||
{
|
||||
try
|
||||
{
|
||||
var startTime = DateTime.UtcNow;
|
||||
while ((DateTime.UtcNow - startTime).TotalMilliseconds < maxWaitMs)
|
||||
{
|
||||
var newMinutesAgo = await ExtractMinutesAgoFromDisplayAsync(page);
|
||||
if (newMinutesAgo.HasValue && newMinutesAgo.Value != currentMinutesAgo)
|
||||
{
|
||||
return;
|
||||
}
|
||||
await page.WaitForTimeoutAsync(200);
|
||||
}
|
||||
Logger.LogDebug("Display label did not change from {CurrentMinutesAgo} within {MaxWaitMs}ms", currentMinutesAgo, maxWaitMs);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogDebug(ex, "Error waiting for display label to change");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task DismissModalOverlaysAsync(IPage page)
|
||||
{
|
||||
try
|
||||
{
|
||||
var hasModal = await page.EvaluateAsync<bool>(JavaScriptTemplates.CheckModalOverlay);
|
||||
|
||||
if (!hasModal)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Logger.LogDebug("Modal overlay detected, dismissing");
|
||||
|
||||
await page.Keyboard.PressAsync("Escape");
|
||||
await page.WaitForTimeoutAsync(200);
|
||||
|
||||
var stillVisible = await page.EvaluateAsync<bool>(JavaScriptTemplates.CheckModalStillVisible);
|
||||
|
||||
if (stillVisible)
|
||||
{
|
||||
try
|
||||
{
|
||||
var mapContainer = SelectorService.GetLocator(page, Selectors.MapContainer);
|
||||
await mapContainer.ClickAsync(new LocatorClickOptions { Force = true });
|
||||
await page.WaitForTimeoutAsync(100);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Ignore if click fails
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogDebug(ex, "Error dismissing modal overlay, continuing");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task CaptureMapScreenshotAsync(IPage page, ILocator mapContainer, string outputPath, Clip containerClip)
|
||||
{
|
||||
if (containerClip == null || containerClip.Width <= 0 || containerClip.Height <= 0)
|
||||
{
|
||||
Logger.LogError("Invalid container bounds: X={X}, Y={Y}, Width={Width}, Height={Height}",
|
||||
containerClip?.X ?? 0, containerClip?.Y ?? 0, containerClip?.Width ?? 0, containerClip?.Height ?? 0);
|
||||
throw new Exception($"Invalid container bounds: {containerClip?.Width ?? 0}x{containerClip?.Height ?? 0}");
|
||||
}
|
||||
|
||||
Clip cropArea;
|
||||
try
|
||||
{
|
||||
cropArea = CalculateCropArea(containerClip);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogWarning(ex, "Failed to calculate crop area, using full container bounds");
|
||||
cropArea = containerClip;
|
||||
}
|
||||
|
||||
var viewportSize = page.ViewportSize;
|
||||
int? viewportWidth = viewportSize?.Width;
|
||||
int? viewportHeight = viewportSize?.Height;
|
||||
|
||||
if (viewportWidth == null || viewportHeight == null)
|
||||
{
|
||||
try
|
||||
{
|
||||
var viewportJson = await page.EvaluateAsync<string>(JavaScriptTemplates.GetViewportSize);
|
||||
if (!string.IsNullOrEmpty(viewportJson))
|
||||
{
|
||||
using var doc = System.Text.Json.JsonDocument.Parse(viewportJson);
|
||||
var root = doc.RootElement;
|
||||
if (root.TryGetProperty("width", out var widthProp) && root.TryGetProperty("height", out var heightProp))
|
||||
{
|
||||
if (widthProp.TryGetInt32(out var width) && heightProp.TryGetInt32(out var height))
|
||||
{
|
||||
viewportWidth = width;
|
||||
viewportHeight = height;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogWarning(ex, "Failed to get viewport size from page evaluation");
|
||||
}
|
||||
|
||||
if (viewportWidth == null || viewportHeight == null)
|
||||
{
|
||||
viewportWidth = (int)containerClip.Width;
|
||||
viewportHeight = (int)containerClip.Height;
|
||||
}
|
||||
}
|
||||
|
||||
if (viewportWidth.HasValue && viewportHeight.HasValue)
|
||||
{
|
||||
if (cropArea.X < 0)
|
||||
{
|
||||
cropArea = new Clip { X = 0, Y = cropArea.Y, Width = cropArea.Width + cropArea.X, Height = cropArea.Height };
|
||||
}
|
||||
if (cropArea.Y < 0)
|
||||
{
|
||||
cropArea = new Clip { X = cropArea.X, Y = 0, Width = cropArea.Width, Height = cropArea.Height + cropArea.Y };
|
||||
}
|
||||
|
||||
if (cropArea.X + cropArea.Width > viewportWidth.Value)
|
||||
{
|
||||
var newWidth = viewportWidth.Value - cropArea.X;
|
||||
cropArea = new Clip { X = cropArea.X, Y = cropArea.Y, Width = newWidth, Height = cropArea.Height };
|
||||
}
|
||||
if (cropArea.Y + cropArea.Height > viewportHeight.Value)
|
||||
{
|
||||
var newHeight = viewportHeight.Value - cropArea.Y;
|
||||
cropArea = new Clip { X = cropArea.X, Y = cropArea.Y, Width = cropArea.Width, Height = newHeight };
|
||||
}
|
||||
}
|
||||
|
||||
if (cropArea.Width <= 0 || cropArea.Height <= 0)
|
||||
{
|
||||
Logger.LogError("Invalid crop dimensions after validation: {Width}x{Height}, using full container", cropArea.Width, cropArea.Height);
|
||||
cropArea = containerClip;
|
||||
}
|
||||
|
||||
if (cropArea.Width <= 0 || cropArea.Height <= 0)
|
||||
{
|
||||
throw new Exception($"Cannot create valid crop area. Container: {containerClip.Width}x{containerClip.Height}, Viewport: {viewportWidth ?? 0}x{viewportHeight ?? 0}");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await page.WaitForLoadStateAsync(LoadState.NetworkIdle, new PageWaitForLoadStateOptions { Timeout = 5000 });
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Continue if network idle timeout
|
||||
}
|
||||
|
||||
await DismissModalOverlaysAsync(page);
|
||||
|
||||
await page.ScreenshotAsync(new PageScreenshotOptions
|
||||
{
|
||||
Path = outputPath,
|
||||
Clip = cropArea,
|
||||
Type = ScreenshotType.Png,
|
||||
Animations = ScreenshotAnimations.Disabled
|
||||
});
|
||||
|
||||
Logger.LogDebug("Screenshot saved: {Path} (crop: {X},{Y} {Width}x{Height})",
|
||||
outputPath, cropArea.X, cropArea.Y, cropArea.Width, cropArea.Height);
|
||||
}
|
||||
|
||||
private Clip CalculateCropArea(Clip containerClip)
|
||||
{
|
||||
var x = containerClip.X + _cropConfig.X;
|
||||
var y = containerClip.Y + _cropConfig.Y;
|
||||
var width = Math.Max(0, containerClip.Width - _cropConfig.X - _cropConfig.RightOffset);
|
||||
var height = _cropConfig.Height ?? Math.Max(0, containerClip.Height - _cropConfig.Y);
|
||||
|
||||
if (x < containerClip.X || y < containerClip.Y)
|
||||
{
|
||||
x = containerClip.X;
|
||||
y = containerClip.Y;
|
||||
}
|
||||
|
||||
var maxWidth = containerClip.Width - (x - containerClip.X);
|
||||
var maxHeight = containerClip.Height - (y - containerClip.Y);
|
||||
|
||||
if (width > maxWidth)
|
||||
{
|
||||
width = maxWidth;
|
||||
}
|
||||
|
||||
if (height > maxHeight)
|
||||
{
|
||||
height = maxHeight;
|
||||
}
|
||||
|
||||
if (width <= 0 || height <= 0)
|
||||
{
|
||||
throw new Exception($"Invalid crop dimensions: {width}x{height}");
|
||||
}
|
||||
|
||||
return new Clip
|
||||
{
|
||||
X = x,
|
||||
Y = y,
|
||||
Width = width,
|
||||
Height = height
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
using BomLocalService.Services.Interfaces;
|
||||
using BomLocalService.Services.Scraping;
|
||||
using Microsoft.Playwright;
|
||||
|
||||
namespace BomLocalService.Services.Scraping.Steps.Map;
|
||||
|
||||
public class CalculateMapBoundsStep : BaseScrapingStep
|
||||
{
|
||||
public override string Name => "CalculateMapBounds";
|
||||
public override string[] Prerequisites => new[] { "ResetToFirstFrame" };
|
||||
|
||||
public CalculateMapBoundsStep(
|
||||
ILogger<CalculateMapBoundsStep> logger,
|
||||
ISelectorService selectorService,
|
||||
IDebugService debugService,
|
||||
IConfiguration configuration)
|
||||
: base(logger, selectorService, debugService, configuration)
|
||||
{
|
||||
}
|
||||
|
||||
public override bool CanExecute(ScrapingContext context)
|
||||
{
|
||||
return context.IsMapReady;
|
||||
}
|
||||
|
||||
public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
Logger.LogInformation("Step {Step}: Preparing map container for screenshot", Name);
|
||||
|
||||
var mapContainer = SelectorService.GetLocator(context.Page, Selectors.MapContainer);
|
||||
await mapContainer.WaitForAsync(new LocatorWaitForOptions { Timeout = 10000 });
|
||||
|
||||
await context.Page.WaitForFunctionAsync(
|
||||
JavaScriptTemplates.WaitForMapContainer,
|
||||
new PageWaitForFunctionOptions { Timeout = 10000 });
|
||||
|
||||
var boundingBox = await mapContainer.BoundingBoxAsync();
|
||||
if (boundingBox == null || boundingBox.Width <= 0 || boundingBox.Height <= 0)
|
||||
{
|
||||
var errorMsg = $"Map container has invalid bounds: {boundingBox?.Width ?? 0}x{boundingBox?.Height ?? 0}";
|
||||
Logger.LogError("Step {Step}: {Error}", Name, errorMsg);
|
||||
await SaveErrorDebugAsync(context, errorMsg, cancellationToken);
|
||||
return ScrapingStepResult.Failed(errorMsg);
|
||||
}
|
||||
|
||||
var containerClip = new Clip
|
||||
{
|
||||
X = boundingBox.X,
|
||||
Y = boundingBox.Y,
|
||||
Width = boundingBox.Width,
|
||||
Height = boundingBox.Height
|
||||
};
|
||||
|
||||
context.MapContainer = mapContainer;
|
||||
context.MapBoundingBox = containerClip;
|
||||
|
||||
return ScrapingStepResult.Successful();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogError(ex, "Step {Step} failed", Name);
|
||||
await SaveErrorDebugAsync(context, $"Failed to calculate map bounds: {ex.Message}", cancellationToken);
|
||||
return ScrapingStepResult.Failed($"Failed to calculate map bounds: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
using BomLocalService.Services.Interfaces;
|
||||
using BomLocalService.Services.Scraping;
|
||||
using Microsoft.Playwright;
|
||||
|
||||
namespace BomLocalService.Services.Scraping.Steps.Map;
|
||||
|
||||
public class PauseRadarStep : BaseScrapingStep
|
||||
{
|
||||
public override string Name => "PauseRadar";
|
||||
public override string[] Prerequisites => new[] { "WaitForMapReady" };
|
||||
|
||||
public PauseRadarStep(
|
||||
ILogger<PauseRadarStep> logger,
|
||||
ISelectorService selectorService,
|
||||
IDebugService debugService,
|
||||
IConfiguration configuration)
|
||||
: base(logger, selectorService, debugService, configuration)
|
||||
{
|
||||
}
|
||||
|
||||
public override bool CanExecute(ScrapingContext context)
|
||||
{
|
||||
return context.IsMapReady;
|
||||
}
|
||||
|
||||
public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
Logger.LogInformation("Step {Step}: Checking if radar loop is paused", Name);
|
||||
|
||||
var playPauseButton = SelectorService.GetLocator(context.Page, Selectors.PlayPauseButton);
|
||||
await playPauseButton.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000 });
|
||||
|
||||
var buttonLabel = await playPauseButton.Locator(Selectors.PlayPauseLabel.Selectors[0]).TextContentAsync();
|
||||
if (buttonLabel?.Trim().Equals(TextPatterns.ExpectedTexts["PauseButtonLabel"], StringComparison.OrdinalIgnoreCase) == true)
|
||||
{
|
||||
Logger.LogInformation("Step {Step}: Radar is playing, pausing it", Name);
|
||||
await playPauseButton.ClickAsync();
|
||||
await context.Page.WaitForTimeoutAsync(300);
|
||||
|
||||
buttonLabel = await playPauseButton.Locator(Selectors.PlayPauseLabel.Selectors[0]).TextContentAsync();
|
||||
if (buttonLabel?.Trim().Equals(TextPatterns.ExpectedTexts["PlayButtonLabel"], StringComparison.OrdinalIgnoreCase) != true)
|
||||
{
|
||||
Logger.LogWarning("Step {Step}: Radar may not be paused after click, continuing anyway", Name);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Logger.LogInformation("Step {Step}: Radar is already paused", Name);
|
||||
}
|
||||
|
||||
await SaveDebugAsync(context, 8, "radar_paused", cancellationToken);
|
||||
|
||||
context.CurrentState = PageState.SlideshowPaused;
|
||||
return ScrapingStepResult.Successful();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogError(ex, "Step {Step} failed", Name);
|
||||
await SaveErrorDebugAsync(context, $"Failed to pause radar: {ex.Message}", cancellationToken);
|
||||
return ScrapingStepResult.Failed($"Failed to pause radar: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
using BomLocalService.Services.Interfaces;
|
||||
using BomLocalService.Services.Scraping;
|
||||
using Microsoft.Playwright;
|
||||
|
||||
namespace BomLocalService.Services.Scraping.Steps.Map;
|
||||
|
||||
public class ResetToFirstFrameStep : BaseScrapingStep
|
||||
{
|
||||
public override string Name => "ResetToFirstFrame";
|
||||
public override string[] Prerequisites => new[] { "PauseRadar" };
|
||||
|
||||
public ResetToFirstFrameStep(
|
||||
ILogger<ResetToFirstFrameStep> logger,
|
||||
ISelectorService selectorService,
|
||||
IDebugService debugService,
|
||||
IConfiguration configuration)
|
||||
: base(logger, selectorService, debugService, configuration)
|
||||
{
|
||||
}
|
||||
|
||||
public override bool CanExecute(ScrapingContext context)
|
||||
{
|
||||
return context.CurrentState >= PageState.SlideshowPaused;
|
||||
}
|
||||
|
||||
public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
Logger.LogInformation("Step {Step}: Resetting to first frame (frame 0)", Name);
|
||||
|
||||
try
|
||||
{
|
||||
// Try JavaScript click first to avoid thumb interception
|
||||
var clicked = await context.Page.EvaluateAsync<bool>(@"() => {
|
||||
const segment = document.querySelector('[data-testid=""bom-scrub-segment""][data-id=""0""]');
|
||||
if (segment) {
|
||||
segment.click();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}");
|
||||
|
||||
if (clicked)
|
||||
{
|
||||
await context.Page.WaitForTimeoutAsync(500);
|
||||
Logger.LogInformation("Step {Step}: Successfully clicked frame 0 segment via JavaScript", Name);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Fallback to locator click with force
|
||||
var firstFrameSegment = context.Page.Locator("[data-testid='bom-scrub-segment'][data-id='0']").First;
|
||||
await firstFrameSegment.WaitForAsync(new LocatorWaitForOptions { Timeout = 5000 });
|
||||
await firstFrameSegment.ClickAsync(new LocatorClickOptions { Force = true });
|
||||
await context.Page.WaitForTimeoutAsync(500);
|
||||
Logger.LogInformation("Step {Step}: Successfully clicked frame 0 segment via force click", Name);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogWarning(ex, "Step {Step}: Failed to click first frame segment, continuing anyway", Name);
|
||||
}
|
||||
|
||||
await SaveDebugAsync(context, 9, "frame_0_selected", cancellationToken);
|
||||
|
||||
// Verify scrubber is at position 0
|
||||
Logger.LogInformation("Step {Step}: Verifying scrubber is at position 0", Name);
|
||||
try
|
||||
{
|
||||
var activeSegment = await context.Page.EvaluateAsync<bool>(JavaScriptTemplates.CheckActiveFrameSegment);
|
||||
|
||||
if (activeSegment)
|
||||
{
|
||||
Logger.LogInformation("Step {Step}: Scrubber confirmed at position 0", Name);
|
||||
}
|
||||
else
|
||||
{
|
||||
Logger.LogDebug("Step {Step}: Could not confirm scrubber position, but continuing", Name);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogDebug(ex, "Step {Step}: Scrubber position verification failed, continuing anyway", Name);
|
||||
}
|
||||
|
||||
await SaveDebugAsync(context, 10, "scrubber_at_position_0", cancellationToken);
|
||||
|
||||
// Wait for frame 0 tiles to fully render
|
||||
var tileRenderWaitMs = Configuration.GetValue<int>("Screenshot:TileRenderWaitMs", 5000);
|
||||
await context.Page.WaitForTimeoutAsync(tileRenderWaitMs);
|
||||
|
||||
context.CurrentState = PageState.Frame0Selected;
|
||||
return ScrapingStepResult.Successful();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogError(ex, "Step {Step} failed", Name);
|
||||
await SaveErrorDebugAsync(context, $"Failed to reset to first frame: {ex.Message}", cancellationToken);
|
||||
return ScrapingStepResult.Failed($"Failed to reset to first frame: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
using BomLocalService.Services.Interfaces;
|
||||
using BomLocalService.Services.Scraping;
|
||||
using Microsoft.Playwright;
|
||||
|
||||
namespace BomLocalService.Services.Scraping.Steps.Map;
|
||||
|
||||
public class WaitForMapReadyStep : BaseScrapingStep
|
||||
{
|
||||
private readonly int _tileRenderWaitMs;
|
||||
|
||||
public override string Name => "WaitForMapReady";
|
||||
public override string[] Prerequisites => new[] { "ClickRadarLink" };
|
||||
|
||||
public WaitForMapReadyStep(
|
||||
ILogger<WaitForMapReadyStep> logger,
|
||||
ISelectorService selectorService,
|
||||
IDebugService debugService,
|
||||
IConfiguration configuration)
|
||||
: base(logger, selectorService, debugService, configuration)
|
||||
{
|
||||
_tileRenderWaitMs = configuration.GetValue<int>("Screenshot:TileRenderWaitMs", 5000);
|
||||
}
|
||||
|
||||
public override bool CanExecute(ScrapingContext context)
|
||||
{
|
||||
return context.IsRadarPageLoaded;
|
||||
}
|
||||
|
||||
public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
Logger.LogInformation("Step {Step}: Waiting for weather map page to load", Name);
|
||||
|
||||
await context.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15000 });
|
||||
|
||||
Logger.LogInformation("Step {Step}: Waiting for map canvas element to render", Name);
|
||||
var mapCanvas = SelectorService.GetLocator(context.Page, Selectors.MapCanvas);
|
||||
await mapCanvas.WaitForAsync(new LocatorWaitForOptions { Timeout = 15000 });
|
||||
|
||||
await context.Page.WaitForFunctionAsync(
|
||||
JavaScriptTemplates.WaitForMapCanvas,
|
||||
new PageWaitForFunctionOptions { Timeout = 15000 });
|
||||
|
||||
Logger.LogInformation("Step {Step}: Map canvas is ready - waiting for map to render", Name);
|
||||
|
||||
try
|
||||
{
|
||||
await context.Page.WaitForFunctionAsync(
|
||||
JavaScriptTemplates.WaitForEsriView,
|
||||
new PageWaitForFunctionOptions { Timeout = 30000 });
|
||||
Logger.LogInformation("Step {Step}: Esri map view is ready", Name);
|
||||
}
|
||||
catch
|
||||
{
|
||||
Logger.LogInformation("Step {Step}: Esri view ready check timed out, continuing with fixed wait", Name);
|
||||
}
|
||||
|
||||
await context.Page.WaitForTimeoutAsync(_tileRenderWaitMs);
|
||||
await SaveDebugAsync(context, 7, "weather_map_ready", cancellationToken);
|
||||
|
||||
context.CurrentState = PageState.MapReady;
|
||||
return ScrapingStepResult.Successful();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogError(ex, "Step {Step} failed", Name);
|
||||
await SaveErrorDebugAsync(context, $"Failed to wait for map ready: {ex.Message}", cancellationToken);
|
||||
return ScrapingStepResult.Failed($"Failed to wait for map ready: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
using BomLocalService.Services.Interfaces;
|
||||
using BomLocalService.Services.Scraping;
|
||||
|
||||
namespace BomLocalService.Services.Scraping.Steps.Metadata;
|
||||
|
||||
public class ExtractMetadataStep : BaseScrapingStep
|
||||
{
|
||||
private readonly ITimeParsingService _timeParsingService;
|
||||
|
||||
public override string Name => "ExtractMetadata";
|
||||
public override string[] Prerequisites => new[] { "ResetToFirstFrame" };
|
||||
|
||||
public ExtractMetadataStep(
|
||||
ILogger<ExtractMetadataStep> logger,
|
||||
ISelectorService selectorService,
|
||||
IDebugService debugService,
|
||||
IConfiguration configuration,
|
||||
ITimeParsingService timeParsingService)
|
||||
: base(logger, selectorService, debugService, configuration)
|
||||
{
|
||||
_timeParsingService = timeParsingService;
|
||||
}
|
||||
|
||||
public override bool CanExecute(ScrapingContext context)
|
||||
{
|
||||
return context.IsMapReady;
|
||||
}
|
||||
|
||||
public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
Logger.LogInformation("Step {Step}: Extracting metadata and frame information", Name);
|
||||
|
||||
var lastUpdatedInfo = await _timeParsingService.ExtractLastUpdatedInfoAsync(context.Page);
|
||||
context.LastUpdatedInfo = lastUpdatedInfo;
|
||||
|
||||
// Extract frame info
|
||||
try
|
||||
{
|
||||
var frameInfo = await context.Page.EvaluateAsync<object[]>(JavaScriptTemplates.ExtractFrameInfo);
|
||||
var result = new List<(int index, int minutesAgo)>();
|
||||
for (int i = 0; i < 7; i++)
|
||||
{
|
||||
var minutesAgo = 40 - (i * 5);
|
||||
result.Add((i, minutesAgo));
|
||||
}
|
||||
context.FrameInfo = result;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogWarning(ex, "Step {Step}: Failed to extract frame info, using defaults", Name);
|
||||
context.FrameInfo = Enumerable.Range(0, 7)
|
||||
.Select(i => (i, 40 - (i * 5)))
|
||||
.ToList();
|
||||
}
|
||||
|
||||
return ScrapingStepResult.Successful();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogError(ex, "Step {Step} failed", Name);
|
||||
await SaveErrorDebugAsync(context, $"Failed to extract metadata: {ex.Message}", cancellationToken);
|
||||
return ScrapingStepResult.Failed($"Failed to extract metadata: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
using BomLocalService.Services.Interfaces;
|
||||
using BomLocalService.Services.Scraping;
|
||||
using Microsoft.Playwright;
|
||||
|
||||
namespace BomLocalService.Services.Scraping.Steps.Navigation;
|
||||
|
||||
public class ClickRadarLinkStep : BaseScrapingStep
|
||||
{
|
||||
public override string Name => "ClickRadarLink";
|
||||
public override string[] Prerequisites => new[] { "SelectSearchResult" };
|
||||
|
||||
public ClickRadarLinkStep(
|
||||
ILogger<ClickRadarLinkStep> logger,
|
||||
ISelectorService selectorService,
|
||||
IDebugService debugService,
|
||||
IConfiguration configuration)
|
||||
: base(logger, selectorService, debugService, configuration)
|
||||
{
|
||||
}
|
||||
|
||||
public override bool CanExecute(ScrapingContext context)
|
||||
{
|
||||
return context.IsForecastPageLoaded;
|
||||
}
|
||||
|
||||
public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
Logger.LogInformation("Step {Step}: Looking for 'Rain radar and weather map' link", Name);
|
||||
|
||||
var radarLink = await SelectorService.FindElementAsync(context.Page, Selectors.RadarLink, cancellationToken);
|
||||
|
||||
if (radarLink == null)
|
||||
{
|
||||
var errorMsg = Selectors.RadarLink.ErrorMessage ?? $"Could not find 'Rain radar and weather map' link for {context.Suburb}, {context.State}";
|
||||
await SaveErrorDebugAsync(context, errorMsg, cancellationToken);
|
||||
return ScrapingStepResult.Failed(errorMsg);
|
||||
}
|
||||
|
||||
context.RadarLink = radarLink;
|
||||
await radarLink.ClickAsync();
|
||||
await SaveDebugAsync(context, 6, "radar_link_clicked", cancellationToken);
|
||||
|
||||
context.CurrentState = PageState.RadarPageLoaded;
|
||||
return ScrapingStepResult.Successful();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogError(ex, "Step {Step} failed", Name);
|
||||
await SaveErrorDebugAsync(context, $"Failed to click radar link: {ex.Message}", cancellationToken);
|
||||
return ScrapingStepResult.Failed($"Failed to click radar link: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
using BomLocalService.Services.Interfaces;
|
||||
using BomLocalService.Services.Scraping;
|
||||
using Microsoft.Playwright;
|
||||
|
||||
namespace BomLocalService.Services.Scraping.Steps.Navigation;
|
||||
|
||||
public class ClickSearchButtonStep : BaseScrapingStep
|
||||
{
|
||||
public override string Name => "ClickSearchButton";
|
||||
public override string[] Prerequisites => new[] { "NavigateHomepage" };
|
||||
|
||||
public ClickSearchButtonStep(
|
||||
ILogger<ClickSearchButtonStep> logger,
|
||||
ISelectorService selectorService,
|
||||
IDebugService debugService,
|
||||
IConfiguration configuration)
|
||||
: base(logger, selectorService, debugService, configuration)
|
||||
{
|
||||
}
|
||||
|
||||
public override bool CanExecute(ScrapingContext context)
|
||||
{
|
||||
return context.IsHomepageLoaded;
|
||||
}
|
||||
|
||||
public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
Logger.LogInformation("Step {Step}: Clicking 'Search for a location' button", Name);
|
||||
|
||||
var searchButton = await SelectorService.FindElementAsync(context.Page, Selectors.SearchButton, cancellationToken);
|
||||
|
||||
if (searchButton == null)
|
||||
{
|
||||
var errorMsg = Selectors.SearchButton.ErrorMessage ?? "Could not find 'Search for a location' button on BOM homepage.";
|
||||
await SaveErrorDebugAsync(context, errorMsg, cancellationToken);
|
||||
return ScrapingStepResult.Failed(errorMsg);
|
||||
}
|
||||
|
||||
context.SearchButton = searchButton;
|
||||
await searchButton.ClickAsync();
|
||||
|
||||
// Wait for search input to appear
|
||||
var searchInputReady = SelectorService.GetLocator(context.Page, Selectors.SearchInput);
|
||||
await searchInputReady.WaitForAsync(new LocatorWaitForOptions
|
||||
{
|
||||
Timeout = Selectors.SearchInput.TimeoutMs,
|
||||
State = WaitForSelectorState.Visible
|
||||
});
|
||||
|
||||
await SaveDebugAsync(context, 2, "search_button_clicked", cancellationToken);
|
||||
|
||||
context.CurrentState = PageState.SearchModalOpen;
|
||||
return ScrapingStepResult.Successful();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogError(ex, "Step {Step} failed", Name);
|
||||
await SaveErrorDebugAsync(context, $"Failed to click search button: {ex.Message}", cancellationToken);
|
||||
return ScrapingStepResult.Failed($"Failed to click search button: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
using BomLocalService.Services.Interfaces;
|
||||
using BomLocalService.Services.Scraping;
|
||||
using Microsoft.Playwright;
|
||||
|
||||
namespace BomLocalService.Services.Scraping.Steps.Navigation;
|
||||
|
||||
public class NavigateHomepageStep : BaseScrapingStep
|
||||
{
|
||||
private readonly string _baseUrl;
|
||||
|
||||
public override string Name => "NavigateHomepage";
|
||||
public override string[] Prerequisites => Array.Empty<string>();
|
||||
|
||||
public NavigateHomepageStep(
|
||||
ILogger<NavigateHomepageStep> logger,
|
||||
ISelectorService selectorService,
|
||||
IDebugService debugService,
|
||||
IConfiguration configuration)
|
||||
: base(logger, selectorService, debugService, configuration)
|
||||
{
|
||||
_baseUrl = configuration.GetValue<string>("Scraping:BaseUrl") ?? "https://www.bom.gov.au/";
|
||||
}
|
||||
|
||||
public override bool CanExecute(ScrapingContext context)
|
||||
{
|
||||
return context.CurrentState == PageState.Initial;
|
||||
}
|
||||
|
||||
public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
Logger.LogInformation("Step {Step}: Navigating to BOM homepage", Name);
|
||||
|
||||
await context.Page.GotoAsync(_baseUrl, new PageGotoOptions
|
||||
{
|
||||
WaitUntil = WaitUntilState.DOMContentLoaded,
|
||||
Timeout = 30000
|
||||
});
|
||||
|
||||
// Wait for search button to be ready
|
||||
var searchButtonReady = SelectorService.GetLocator(context.Page, Selectors.SearchButton);
|
||||
await searchButtonReady.WaitForAsync(new LocatorWaitForOptions
|
||||
{
|
||||
Timeout = Selectors.SearchButton.TimeoutMs,
|
||||
State = WaitForSelectorState.Visible
|
||||
});
|
||||
|
||||
await SaveDebugAsync(context, 1, "homepage_loaded", cancellationToken);
|
||||
|
||||
context.CurrentState = PageState.HomepageLoaded;
|
||||
return ScrapingStepResult.Successful();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogError(ex, "Step {Step} failed", Name);
|
||||
await SaveErrorDebugAsync(context, $"Failed to navigate to homepage: {ex.Message}", cancellationToken);
|
||||
return ScrapingStepResult.Failed($"Failed to navigate to homepage: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
using BomLocalService.Services.Interfaces;
|
||||
using BomLocalService.Services.Scraping;
|
||||
using Microsoft.Playwright;
|
||||
|
||||
namespace BomLocalService.Services.Scraping.Steps.Search;
|
||||
|
||||
public class FillSearchInputStep : BaseScrapingStep
|
||||
{
|
||||
public override string Name => "FillSearchInput";
|
||||
public override string[] Prerequisites => new[] { "ClickSearchButton" };
|
||||
|
||||
public FillSearchInputStep(
|
||||
ILogger<FillSearchInputStep> logger,
|
||||
ISelectorService selectorService,
|
||||
IDebugService debugService,
|
||||
IConfiguration configuration)
|
||||
: base(logger, selectorService, debugService, configuration)
|
||||
{
|
||||
}
|
||||
|
||||
public override bool CanExecute(ScrapingContext context)
|
||||
{
|
||||
return context.IsSearchModalOpen;
|
||||
}
|
||||
|
||||
public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
Logger.LogInformation("Step {Step}: Searching for suburb: {Suburb}", Name, context.Suburb);
|
||||
|
||||
var searchInput = SelectorService.GetLocator(context.Page, Selectors.SearchInput);
|
||||
await searchInput.FillAsync(context.Suburb);
|
||||
|
||||
context.SearchInput = searchInput;
|
||||
await SaveDebugAsync(context, 3, "search_input_filled", cancellationToken);
|
||||
|
||||
return ScrapingStepResult.Successful();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogError(ex, "Step {Step} failed", Name);
|
||||
await SaveErrorDebugAsync(context, $"Failed to fill search input: {ex.Message}", cancellationToken);
|
||||
return ScrapingStepResult.Failed($"Failed to fill search input: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
using BomLocalService.Services.Interfaces;
|
||||
using BomLocalService.Services.Scraping;
|
||||
using BomLocalService.Utilities;
|
||||
using Microsoft.Playwright;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace BomLocalService.Services.Scraping.Steps.Search;
|
||||
|
||||
public class SelectSearchResultStep : BaseScrapingStep
|
||||
{
|
||||
public override string Name => "SelectSearchResult";
|
||||
public override string[] Prerequisites => new[] { "WaitForSearchResults" };
|
||||
|
||||
public SelectSearchResultStep(
|
||||
ILogger<SelectSearchResultStep> logger,
|
||||
ISelectorService selectorService,
|
||||
IDebugService debugService,
|
||||
IConfiguration configuration)
|
||||
: base(logger, selectorService, debugService, configuration)
|
||||
{
|
||||
}
|
||||
|
||||
public override bool CanExecute(ScrapingContext context)
|
||||
{
|
||||
return context.CurrentState >= PageState.SearchResultsVisible;
|
||||
}
|
||||
|
||||
public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
Logger.LogInformation("Step {Step}: Looking for matching search result for {Suburb}, {State}", Name, context.Suburb, context.State);
|
||||
|
||||
var suburbLower = context.Suburb.ToLower().Trim();
|
||||
var stateLower = context.State.ToLower().Trim();
|
||||
|
||||
// Get the actual count from the summary element
|
||||
var resultsTitle = SelectorService.GetLocator(context.Page, Selectors.ResultsTitle);
|
||||
var summaryText = await resultsTitle.TextContentAsync();
|
||||
int? actualCount = null;
|
||||
|
||||
if (!string.IsNullOrEmpty(summaryText))
|
||||
{
|
||||
var countMatch = Regex.Match(summaryText, TextPatterns.ResultsCountPattern, RegexOptions.IgnoreCase);
|
||||
if (countMatch.Success && countMatch.Groups.Count >= 3 && countMatch.Groups[2].Success)
|
||||
{
|
||||
if (int.TryParse(countMatch.Groups[2].Value, out var total))
|
||||
{
|
||||
actualCount = total;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch all result data
|
||||
List<(string name, string desc, string fullText)> results = new();
|
||||
try
|
||||
{
|
||||
var resultData = await context.Page.EvaluateAsync<string[][]>(JavaScriptTemplates.ExtractSearchResults);
|
||||
|
||||
results = resultData.Select(arr => (
|
||||
name: arr.Length > 0 ? arr[0] : "",
|
||||
desc: arr.Length > 1 ? arr[1] : "",
|
||||
fullText: arr.Length > 2 ? arr[2] : ""
|
||||
)).ToList();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogWarning(ex, "Failed to extract structured result data, falling back to text content");
|
||||
var resultTexts = await context.Page.EvaluateAsync<string[]>(JavaScriptTemplates.ExtractSearchResultsFallback);
|
||||
results = resultTexts.Select(text => (name: "", desc: "", fullText: text)).ToList();
|
||||
}
|
||||
|
||||
if (actualCount.HasValue)
|
||||
{
|
||||
Logger.LogInformation("Found {Count} location results (summary: {Summary})", actualCount.Value, summaryText?.Trim());
|
||||
if (results.Count > actualCount.Value)
|
||||
{
|
||||
results = results.Take(actualCount.Value).ToList();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Logger.LogInformation("Found {Count} search results", results.Count);
|
||||
}
|
||||
|
||||
int? matchingIndex = null;
|
||||
int bestMatchScore = -1;
|
||||
|
||||
for (int i = 0; i < results.Count; i++)
|
||||
{
|
||||
var (name, desc, fullText) = results[i];
|
||||
var nameLower = name.ToLower().Trim();
|
||||
var descLower = desc.ToLower().Trim();
|
||||
var fullTextLower = fullText.ToLower();
|
||||
|
||||
Logger.LogInformation("Checking result {Index}: Name='{Name}', Desc='{Desc}', FullText='{FullText}'",
|
||||
i, name, desc, fullText.Length > 100 ? fullText.Substring(0, 100) + "..." : fullText);
|
||||
|
||||
var matchesSuburb = false;
|
||||
var matchScore = 0;
|
||||
|
||||
if (!string.IsNullOrEmpty(name))
|
||||
{
|
||||
if (nameLower == suburbLower)
|
||||
{
|
||||
matchesSuburb = true;
|
||||
matchScore = 100;
|
||||
}
|
||||
else if (nameLower.StartsWith(suburbLower + " ") || nameLower.StartsWith(suburbLower + "("))
|
||||
{
|
||||
matchesSuburb = true;
|
||||
matchScore = 80;
|
||||
}
|
||||
else if (nameLower.Contains("(" + suburbLower + ")") || nameLower.Contains("(" + suburbLower + " "))
|
||||
{
|
||||
matchesSuburb = true;
|
||||
matchScore = 60;
|
||||
}
|
||||
else if (nameLower.Contains(suburbLower))
|
||||
{
|
||||
matchesSuburb = true;
|
||||
matchScore = 40;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (fullTextLower.Contains(suburbLower))
|
||||
{
|
||||
matchesSuburb = true;
|
||||
matchScore = 20;
|
||||
}
|
||||
}
|
||||
|
||||
var matchesState = false;
|
||||
if (!string.IsNullOrEmpty(desc))
|
||||
{
|
||||
matchesState = StateAbbreviationHelper.MatchesState(descLower, stateLower);
|
||||
}
|
||||
if (!matchesState)
|
||||
{
|
||||
matchesState = StateAbbreviationHelper.MatchesState(fullTextLower, stateLower);
|
||||
}
|
||||
|
||||
Logger.LogInformation("Result {Index}: matchesSuburb={MatchesSuburb} (score={Score}), matchesState={MatchesState}",
|
||||
i, matchesSuburb, matchScore, matchesState);
|
||||
|
||||
if (matchesSuburb && matchesState && matchScore > bestMatchScore)
|
||||
{
|
||||
matchingIndex = i;
|
||||
bestMatchScore = matchScore;
|
||||
Logger.LogInformation("New best match found: {Name} - {Desc} (score: {Score})", name, desc, matchScore);
|
||||
}
|
||||
}
|
||||
|
||||
if (matchingIndex.HasValue)
|
||||
{
|
||||
Logger.LogInformation("Using best matching result at index {Index} with score {Score}", matchingIndex.Value, bestMatchScore);
|
||||
}
|
||||
else
|
||||
{
|
||||
Logger.LogInformation("No exact match found, using first result");
|
||||
}
|
||||
|
||||
context.SearchResults = results;
|
||||
context.SelectedResultIndex = matchingIndex;
|
||||
|
||||
// Click the matching result
|
||||
var resultsList = context.Page.Locator("ul[aria-labelledby='location-results-title']");
|
||||
var allResults = resultsList.Locator("li.bom-linklist__item[role='listitem']");
|
||||
var resultToClick = matchingIndex.HasValue ? allResults.Nth(matchingIndex.Value) : allResults.First;
|
||||
|
||||
await resultToClick.ClickAsync();
|
||||
await SaveDebugAsync(context, 4, "search_result_selected", cancellationToken);
|
||||
|
||||
// Wait for forecast page to load
|
||||
await context.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15000 });
|
||||
var dynamicContentWaitMs = Configuration.GetValue<int>("Screenshot:DynamicContentWaitMs", 2000);
|
||||
await context.Page.WaitForTimeoutAsync(dynamicContentWaitMs);
|
||||
await SaveDebugAsync(context, 5, "forecast_page_loaded", cancellationToken);
|
||||
|
||||
context.CurrentState = PageState.ForecastPageLoaded;
|
||||
return ScrapingStepResult.Successful();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogError(ex, "Step {Step} failed", Name);
|
||||
await SaveErrorDebugAsync(context, $"Failed to select search result: {ex.Message}", cancellationToken);
|
||||
return ScrapingStepResult.Failed($"Failed to select search result: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
using BomLocalService.Services.Interfaces;
|
||||
using BomLocalService.Services.Scraping;
|
||||
using Microsoft.Playwright;
|
||||
|
||||
namespace BomLocalService.Services.Scraping.Steps.Search;
|
||||
|
||||
public class WaitForSearchResultsStep : BaseScrapingStep
|
||||
{
|
||||
public override string Name => "WaitForSearchResults";
|
||||
public override string[] Prerequisites => new[] { "FillSearchInput" };
|
||||
|
||||
public WaitForSearchResultsStep(
|
||||
ILogger<WaitForSearchResultsStep> logger,
|
||||
ISelectorService selectorService,
|
||||
IDebugService debugService,
|
||||
IConfiguration configuration)
|
||||
: base(logger, selectorService, debugService, configuration)
|
||||
{
|
||||
}
|
||||
|
||||
public override bool CanExecute(ScrapingContext context)
|
||||
{
|
||||
return context.IsSearchModalOpen;
|
||||
}
|
||||
|
||||
public override async Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
Logger.LogInformation("Step {Step}: Waiting for autocomplete suggestions", Name);
|
||||
|
||||
await context.Page.WaitForFunctionAsync(
|
||||
JavaScriptTemplates.WaitForSearchResults,
|
||||
new PageWaitForFunctionOptions { Timeout = 10000 });
|
||||
|
||||
context.CurrentState = PageState.SearchResultsVisible;
|
||||
return ScrapingStepResult.Successful();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogError(ex, "Step {Step} failed", Name);
|
||||
await SaveErrorDebugAsync(context, $"Failed to wait for search results: {ex.Message}", cancellationToken);
|
||||
return ScrapingStepResult.Failed($"Failed to wait for search results: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
namespace BomLocalService.Services.Scraping;
|
||||
|
||||
public class WorkflowFactory : IWorkflowFactory
|
||||
{
|
||||
private readonly IServiceProvider _serviceProvider;
|
||||
|
||||
public WorkflowFactory(IServiceProvider serviceProvider)
|
||||
{
|
||||
_serviceProvider = serviceProvider;
|
||||
}
|
||||
|
||||
public IWorkflow<TResponse> GetWorkflow<TResponse>(string name)
|
||||
{
|
||||
return name switch
|
||||
{
|
||||
"RadarScraping" => (IWorkflow<TResponse>)_serviceProvider.GetRequiredService<Workflows.RadarScrapingWorkflow>(),
|
||||
"TemperatureMap" => (IWorkflow<TResponse>)_serviceProvider.GetRequiredService<Workflows.TemperatureMapWorkflow>(),
|
||||
_ => throw new ArgumentException($"Unknown workflow: {name}")
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
using BomLocalService.Models;
|
||||
using BomLocalService.Services.Interfaces;
|
||||
using BomLocalService.Services.Scraping;
|
||||
using BomLocalService.Utilities;
|
||||
|
||||
namespace BomLocalService.Services.Scraping.Workflows;
|
||||
|
||||
public class RadarScrapingWorkflow : IWorkflow<RadarResponse>
|
||||
{
|
||||
private readonly ILogger<RadarScrapingWorkflow> _logger;
|
||||
private readonly IScrapingStepRegistry _stepRegistry;
|
||||
private readonly IConfiguration _configuration;
|
||||
private readonly ICacheService _cacheService;
|
||||
private readonly double _cacheExpirationMinutes;
|
||||
private readonly int _cacheManagementCheckIntervalMinutes;
|
||||
|
||||
public string Name => "RadarScraping";
|
||||
public string Description => "Scrapes radar images for a location";
|
||||
|
||||
// Fixed step sequence - order matters due to dependencies
|
||||
public string[] StepNames => new[]
|
||||
{
|
||||
"NavigateHomepage",
|
||||
"ClickSearchButton",
|
||||
"FillSearchInput",
|
||||
"WaitForSearchResults",
|
||||
"SelectSearchResult",
|
||||
"ClickRadarLink",
|
||||
"WaitForMapReady",
|
||||
"PauseRadar",
|
||||
"ResetToFirstFrame",
|
||||
"ExtractMetadata",
|
||||
"CalculateMapBounds",
|
||||
"CaptureFrames"
|
||||
};
|
||||
|
||||
public RadarScrapingWorkflow(
|
||||
ILogger<RadarScrapingWorkflow> logger,
|
||||
IScrapingStepRegistry stepRegistry,
|
||||
IConfiguration configuration,
|
||||
ICacheService cacheService)
|
||||
{
|
||||
_logger = logger;
|
||||
_stepRegistry = stepRegistry;
|
||||
_configuration = configuration;
|
||||
_cacheService = cacheService;
|
||||
_cacheExpirationMinutes = configuration.GetValue<double>("CacheExpirationMinutes", 12.5);
|
||||
_cacheManagementCheckIntervalMinutes = configuration.GetValue<int>("CacheManagement:CheckIntervalMinutes", 5);
|
||||
}
|
||||
|
||||
public async Task<RadarResponse> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
var workflowConfig = _configuration.GetSection($"Scraping:Workflows:{Name}").Get<ScrapingWorkflowConfig>();
|
||||
var workflowStartTime = DateTime.UtcNow;
|
||||
var stepTimings = new List<(string stepName, double durationSeconds)>();
|
||||
|
||||
// Record Initializing phase start for metrics tracking
|
||||
if (!string.IsNullOrEmpty(context.CacheFolderPath))
|
||||
{
|
||||
_cacheService.RecordUpdateProgressByFolder(context.CacheFolderPath, CacheUpdatePhase.Initializing);
|
||||
}
|
||||
|
||||
foreach (var stepName in StepNames)
|
||||
{
|
||||
var stepConfig = workflowConfig?.Steps?.GetValueOrDefault(stepName) ?? new ScrapingStepConfig { Enabled = true };
|
||||
|
||||
if (!stepConfig.Enabled)
|
||||
{
|
||||
_logger.LogInformation("Step {Step} is disabled, skipping", stepName);
|
||||
continue;
|
||||
}
|
||||
|
||||
var step = _stepRegistry.GetStep(stepName);
|
||||
if (step == null)
|
||||
{
|
||||
throw new InvalidOperationException($"Step {stepName} not found in registry");
|
||||
}
|
||||
|
||||
if (!ValidatePrerequisites(step, context))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Step {stepName} prerequisites not met. Required: {string.Join(", ", step.Prerequisites)}");
|
||||
}
|
||||
|
||||
if (!step.CanExecute(context))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Step {stepName} cannot execute in current page state: {context.CurrentState}");
|
||||
}
|
||||
|
||||
var stepStartTime = DateTime.UtcNow;
|
||||
_logger.LogInformation("Executing step {Step}", stepName);
|
||||
|
||||
var result = await step.ExecuteAsync(context, cancellationToken);
|
||||
|
||||
var stepDuration = (DateTime.UtcNow - stepStartTime).TotalSeconds;
|
||||
stepTimings.Add((stepName, stepDuration));
|
||||
|
||||
// Record step timing in metrics
|
||||
_cacheService.RecordStepCompletion(stepName, stepDuration);
|
||||
|
||||
// Get historical average for comparison
|
||||
var avgDuration = _cacheService.GetAverageStepDuration(stepName);
|
||||
if (avgDuration > 0)
|
||||
{
|
||||
var diff = stepDuration - avgDuration;
|
||||
var diffPercent = (diff / avgDuration) * 100;
|
||||
var isSlow = stepDuration > avgDuration * 1.5; // 50% slower than average
|
||||
|
||||
if (isSlow)
|
||||
{
|
||||
_logger.LogWarning("Step {Step} took significantly longer than average: {Duration:F2}s (avg: {Avg:F2}s, {Diff:+#.##}s, {DiffPercent:+#0.#}% slower)",
|
||||
stepName, stepDuration, avgDuration, diff, diffPercent);
|
||||
}
|
||||
else if (Math.Abs(diffPercent) > 20) // More than 20% difference (faster or slower)
|
||||
{
|
||||
_logger.LogInformation("Step {Step} completed in {Duration:F2}s (avg: {Avg:F2}s, {Diff:+#.##;-#.##}s, {DiffPercent:+#0.#;-#0.#}%)",
|
||||
stepName, stepDuration, avgDuration, diff, diffPercent);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogInformation("Step {Step} completed in {Duration:F2}s (avg: {Avg:F2}s)", stepName, stepDuration, avgDuration);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogInformation("Step {Step} completed in {Duration:F2}s", stepName, stepDuration);
|
||||
}
|
||||
|
||||
if (!result.Success)
|
||||
{
|
||||
throw new Exception($"Step {stepName} failed: {result.ErrorMessage}");
|
||||
}
|
||||
|
||||
context.CompletedSteps.Add(stepName);
|
||||
}
|
||||
|
||||
var totalDuration = (DateTime.UtcNow - workflowStartTime).TotalSeconds;
|
||||
|
||||
// Check if workflow duration is significantly longer than average
|
||||
var avgTotalDuration = _cacheService.GetAverageTotalDuration();
|
||||
var isWorkflowSlow = avgTotalDuration > 0 && totalDuration > avgTotalDuration * 1.3; // 30% slower than average
|
||||
|
||||
// Log step breakdown
|
||||
var stepBreakdown = string.Join(", ", stepTimings.Select(t => $"{t.stepName}={t.durationSeconds:F2}s"));
|
||||
|
||||
if (isWorkflowSlow)
|
||||
{
|
||||
var workflowDiff = totalDuration - avgTotalDuration;
|
||||
var workflowDiffPercent = (workflowDiff / avgTotalDuration) * 100;
|
||||
_logger.LogWarning("Workflow {Workflow} took significantly longer than average: {TotalDuration:F2}s (avg: {Avg:F2}s, {Diff:+#.##}s, {DiffPercent:+#0.#}% slower) for {Suburb}, {State}",
|
||||
Name, totalDuration, avgTotalDuration, workflowDiff, workflowDiffPercent, context.Suburb, context.State);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogInformation("Workflow {Workflow} completed in {TotalDuration:F2}s for {Suburb}, {State}. Step breakdown: {StepBreakdown}",
|
||||
Name, totalDuration, context.Suburb, context.State, stepBreakdown);
|
||||
}
|
||||
|
||||
// Log step performance summary if we have historical data
|
||||
var stepMetrics = _cacheService.GetStepMetrics();
|
||||
if (stepMetrics.Count > 0)
|
||||
{
|
||||
var slowSteps = stepTimings
|
||||
.Where(t => stepMetrics.ContainsKey(t.stepName) && t.durationSeconds > stepMetrics[t.stepName] * 1.5)
|
||||
.Select(t => $"{t.stepName} ({t.durationSeconds:F2}s vs avg {stepMetrics[t.stepName]:F2}s)")
|
||||
.ToList();
|
||||
|
||||
if (slowSteps.Any())
|
||||
{
|
||||
_logger.LogWarning("Workflow {Workflow} had slower-than-average steps: {SlowSteps}",
|
||||
Name, string.Join(", ", slowSteps));
|
||||
}
|
||||
}
|
||||
|
||||
return BuildResponse(context);
|
||||
}
|
||||
|
||||
private bool ValidatePrerequisites(IScrapingStep step, ScrapingContext context)
|
||||
{
|
||||
return step.Prerequisites.All(prereq => context.CompletedSteps.Contains(prereq));
|
||||
}
|
||||
|
||||
private RadarResponse BuildResponse(ScrapingContext context)
|
||||
{
|
||||
if (context.LastUpdatedInfo == null)
|
||||
{
|
||||
throw new InvalidOperationException("LastUpdatedInfo is required to build response");
|
||||
}
|
||||
|
||||
if (context.Frames == null || context.Frames.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException("Frames are required to build response");
|
||||
}
|
||||
|
||||
var cacheExpiresAt = context.LastUpdatedInfo.ObservationTime.AddMinutes(_cacheExpirationMinutes);
|
||||
|
||||
return ResponseBuilder.CreateRadarResponse(
|
||||
context.CacheFolderPath,
|
||||
context.Frames,
|
||||
_cacheManagementCheckIntervalMinutes,
|
||||
context.LastUpdatedInfo,
|
||||
context.Suburb,
|
||||
context.State,
|
||||
cacheIsValid: true,
|
||||
cacheExpiresAt: cacheExpiresAt,
|
||||
isUpdating: false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
using BomLocalService.Models;
|
||||
using BomLocalService.Services.Interfaces;
|
||||
using BomLocalService.Services.Scraping;
|
||||
|
||||
namespace BomLocalService.Services.Scraping.Workflows;
|
||||
|
||||
/// <summary>
|
||||
/// Future workflow for temperature map scraping (not yet implemented)
|
||||
/// </summary>
|
||||
public class TemperatureMapWorkflow : IWorkflow<RadarResponse>
|
||||
{
|
||||
private readonly ILogger<TemperatureMapWorkflow> _logger;
|
||||
|
||||
public string Name => "TemperatureMap";
|
||||
public string Description => "Scrapes temperature forecast map (future feature)";
|
||||
|
||||
public string[] StepNames => new[]
|
||||
{
|
||||
"NavigateHomepage",
|
||||
"ClickSearchButton",
|
||||
"FillSearchInput",
|
||||
"WaitForSearchResults",
|
||||
"SelectSearchResult",
|
||||
// Future: "ClickTemperatureMapLink",
|
||||
// Future: "WaitForMapReady",
|
||||
// Future: "ExtractMetadata",
|
||||
// Future: "CalculateMapBounds",
|
||||
// Future: "CaptureTemperatureFrames"
|
||||
};
|
||||
|
||||
public TemperatureMapWorkflow(ILogger<TemperatureMapWorkflow> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public Task<RadarResponse> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
throw new NotImplementedException("TemperatureMap workflow is not yet implemented");
|
||||
}
|
||||
}
|
||||
|
||||
+16
-1002
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,60 @@
|
||||
using BomLocalService.Models;
|
||||
using BomLocalService.Services.Interfaces;
|
||||
using Microsoft.Playwright;
|
||||
|
||||
namespace BomLocalService.Services;
|
||||
|
||||
public class SelectorService : ISelectorService
|
||||
{
|
||||
private readonly ILogger<SelectorService> _logger;
|
||||
|
||||
public SelectorService(ILogger<SelectorService> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task<ILocator?> FindElementAsync(IPage page, SelectorConfig config, CancellationToken cancellationToken = default)
|
||||
{
|
||||
foreach (var selector in config.Selectors)
|
||||
{
|
||||
try
|
||||
{
|
||||
var locator = page.Locator(selector).First;
|
||||
if (await locator.IsVisibleAsync())
|
||||
{
|
||||
_logger.LogInformation("Found {Name} with selector: {Selector}", config.Name, selector);
|
||||
return locator;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "Selector {Selector} not found or not visible for {Name}", selector, config.Name);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (config.Required)
|
||||
{
|
||||
var errorMsg = config.ErrorMessage ?? $"Required element {config.Name} not found with any selector";
|
||||
_logger.LogWarning("{Message}", errorMsg);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug("Optional element {Name} not found with any selector", config.Name);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
public ILocator GetLocator(IPage page, SelectorConfig config)
|
||||
{
|
||||
if (config.Selectors.Length == 0)
|
||||
{
|
||||
throw new ArgumentException($"No selectors configured for {config.Name}");
|
||||
}
|
||||
|
||||
// Returns first selector as locator (for cases where we know it exists)
|
||||
return page.Locator(config.Selectors[0]).First;
|
||||
}
|
||||
}
|
||||
|
||||
+197
-2
@@ -25,8 +25,8 @@
|
||||
"IntervalHours": 1
|
||||
},
|
||||
"Screenshot": {
|
||||
"DynamicContentWaitMs": 2000,
|
||||
"TileRenderWaitMs": 5000,
|
||||
"DynamicContentWaitMs": 1500,
|
||||
"TileRenderWaitMs": 3000,
|
||||
"Crop": {
|
||||
"X": 250,
|
||||
"Y": 0,
|
||||
@@ -47,5 +47,200 @@
|
||||
"TimeSeries": {
|
||||
"WarningFolderCount": 200,
|
||||
"MaxTimeRangeHours": null
|
||||
},
|
||||
"Scraping": {
|
||||
"BaseUrl": "https://www.bom.gov.au/",
|
||||
"Timeouts": {
|
||||
"PageNavigation": 30000,
|
||||
"ElementWait": 10000,
|
||||
"SearchResults": 10000,
|
||||
"MapRender": 15000,
|
||||
"TileRender": 5000
|
||||
},
|
||||
"Selectors": {
|
||||
"SearchButton": {
|
||||
"Name": "Search Button",
|
||||
"Selectors": [
|
||||
"button[data-testid='searchLabel']",
|
||||
"button[aria-label='Search for a location']",
|
||||
"button.search-location__trigger-button",
|
||||
"button.bom-button:has(span.bom-button__label:has-text('Search for a location'))",
|
||||
"button:has-text('Search for a location')",
|
||||
"[data-testid='showSearchModal'] button"
|
||||
],
|
||||
"TimeoutMs": 10000,
|
||||
"Required": true,
|
||||
"ErrorMessage": "Could not find 'Search for a location' button on BOM homepage."
|
||||
},
|
||||
"SearchInput": {
|
||||
"Name": "Search Input",
|
||||
"Selectors": ["#search-enter-keyword"],
|
||||
"TimeoutMs": 5000,
|
||||
"Required": true
|
||||
},
|
||||
"SearchResultsList": {
|
||||
"Name": "Search Results List",
|
||||
"Selectors": ["ul[aria-labelledby='location-results-title']"],
|
||||
"TimeoutMs": 10000,
|
||||
"Required": true
|
||||
},
|
||||
"SearchResultItem": {
|
||||
"Name": "Search Result Item",
|
||||
"Selectors": ["li.bom-linklist__item[role='listitem']"],
|
||||
"TimeoutMs": 5000,
|
||||
"Required": true
|
||||
},
|
||||
"LocationName": {
|
||||
"Name": "Location Name",
|
||||
"Selectors": ["[data-testid='location-name']"],
|
||||
"TimeoutMs": 5000,
|
||||
"Required": false
|
||||
},
|
||||
"LocationDescription": {
|
||||
"Name": "Location Description",
|
||||
"Selectors": [".bom-linklist-item__desc"],
|
||||
"TimeoutMs": 5000,
|
||||
"Required": false
|
||||
},
|
||||
"ResultsTitle": {
|
||||
"Name": "Results Title",
|
||||
"Selectors": ["#location-results-title", "[data-testid='location-results-title']"],
|
||||
"TimeoutMs": 5000,
|
||||
"Required": false
|
||||
},
|
||||
"RadarLink": {
|
||||
"Name": "Radar Link",
|
||||
"Selectors": [
|
||||
"a.jump-link.bom-button.bom-button--secondary:has-text('Rain radar and weather map')",
|
||||
".cta.button a.jump-link:has-text('Rain radar and weather map')",
|
||||
"a.jump-link:has-text('Rain radar and weather map')",
|
||||
".cta.button a:has-text('Rain radar and weather map')",
|
||||
"a.bom-button--secondary:has-text('Rain radar and weather map')",
|
||||
"a:has-text('Rain radar and weather map')",
|
||||
"a:has-text('rain radar')",
|
||||
"a[href*='radar']"
|
||||
],
|
||||
"TimeoutMs": 10000,
|
||||
"Required": true,
|
||||
"ErrorMessage": "Could not find 'Rain radar and weather map' link"
|
||||
},
|
||||
"MapCanvas": {
|
||||
"Name": "Map Canvas",
|
||||
"Selectors": [".esri-view-surface canvas"],
|
||||
"TimeoutMs": 15000,
|
||||
"Required": true
|
||||
},
|
||||
"MapContainer": {
|
||||
"Name": "Map Container",
|
||||
"Selectors": [".esri-view-surface"],
|
||||
"TimeoutMs": 10000,
|
||||
"Required": true
|
||||
},
|
||||
"PlayPauseButton": {
|
||||
"Name": "Play Pause Button",
|
||||
"Selectors": ["button[data-testid='bom-time-scrub-play-pause']"],
|
||||
"TimeoutMs": 5000,
|
||||
"Required": true
|
||||
},
|
||||
"PlayPauseLabel": {
|
||||
"Name": "Play Pause Label",
|
||||
"Selectors": [".bom-scrub-action__label"],
|
||||
"TimeoutMs": 2000,
|
||||
"Required": false
|
||||
},
|
||||
"FrameSegment": {
|
||||
"Name": "Frame Segment",
|
||||
"Selectors": ["[data-testid='bom-scrub-segment']"],
|
||||
"TimeoutMs": 5000,
|
||||
"Required": false
|
||||
},
|
||||
"StepForwardButton": {
|
||||
"Name": "Step Forward Button",
|
||||
"Selectors": ["button[data-testid='bom-scrub-utils__right__step-forward']"],
|
||||
"TimeoutMs": 5000,
|
||||
"Required": true
|
||||
},
|
||||
"TimeDisplayLabel": {
|
||||
"Name": "Time Display Label",
|
||||
"Selectors": [".bom-scrub-display-label"],
|
||||
"TimeoutMs": 2000,
|
||||
"Required": false
|
||||
},
|
||||
"ModalOverlay": {
|
||||
"Name": "Modal Overlay",
|
||||
"Selectors": [".bom-modal-overlay--after-open"],
|
||||
"TimeoutMs": 1000,
|
||||
"Required": false
|
||||
},
|
||||
"WeatherMetadata": {
|
||||
"Name": "Weather Metadata",
|
||||
"Selectors": ["section[data-testid='weatherMetadata']", "section[aria-label='Last updated']"],
|
||||
"TimeoutMs": 5000,
|
||||
"Required": false
|
||||
}
|
||||
},
|
||||
"JavaScriptTemplates": {
|
||||
"WaitForSearchResults": "() => { const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=\"listitem\"]')); return results.length > 0 && results.some(r => r.offsetParent !== null); }",
|
||||
"ExtractSearchResults": "() => { const resultsList = document.querySelector('ul[aria-labelledby=\"location-results-title\"]'); if (!resultsList) { console.log('Location results list not found'); return []; } const results = Array.from(resultsList.querySelectorAll('li.bom-linklist__item[role=\"listitem\"]')); console.log('Found', results.length, 'location results'); return results.map((r) => { const nameEl = r.querySelector('[data-testid=\"location-name\"]'); const descEl = r.querySelector('.bom-linklist-item__desc'); const name = nameEl ? (nameEl.textContent || nameEl.innerText || '').trim() : ''; const desc = descEl ? (descEl.textContent || descEl.innerText || '').trim() : ''; const fullText = (r.textContent || r.innerText || '').trim(); console.log('Result:', { hasNameEl: !!nameEl, hasDescEl: !!descEl, name: name, desc: desc }); return [name, desc, fullText]; }); }",
|
||||
"ExtractSearchResultsFallback": "() => { const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=\"listitem\"]')); return results.map(r => r.textContent || ''); }",
|
||||
"WaitForMapCanvas": "() => { const canvas = document.querySelector('.esri-view-surface canvas'); return canvas && canvas.width > 0 && canvas.height > 0 && canvas.offsetWidth > 0 && canvas.offsetHeight > 0; }",
|
||||
"WaitForEsriView": "() => { try { const elements = document.querySelectorAll('.esri-view'); for (let el of elements) { if (el.__view && el.__view.ready) { return true; } } } catch(e) {} return false; }",
|
||||
"CheckActiveFrameSegment": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); const activeSegment = segments.find(s => { const style = window.getComputedStyle(s); return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)'; }); return activeSegment && activeSegment.getAttribute('data-id') === '0'; }",
|
||||
"WaitForMapContainer": "() => { const container = document.querySelector('.esri-view-surface'); return container && container.offsetWidth > 0 && container.offsetHeight > 0; }",
|
||||
"ExtractFrameInfo": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); const now = new Date(); return segments.map((seg, index) => { const ariaLabel = seg.getAttribute('aria-label') || ''; let minutes = null; const timestampMatch = ariaLabel.match(/(?:[A-Za-z]+\\s+)?(\\d{1,2})\\s+([A-Za-z]{3}),?\\s+(\\d{1,2}):(\\d{2})\\s+(am|pm)/i); if (timestampMatch) { try { const day = parseInt(timestampMatch[1]); const monthStr = timestampMatch[2]; const hour12 = parseInt(timestampMatch[3]); const minute = parseInt(timestampMatch[4]); const ampm = timestampMatch[5].toLowerCase(); const monthNames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']; const month = monthNames.indexOf(monthStr.toLowerCase()); if (month >= 0) { let hour24 = hour12; if (ampm === 'pm' && hour12 !== 12) hour24 += 12; if (ampm === 'am' && hour12 === 12) hour24 = 0; const year = now.getFullYear(); const frameTime = new Date(year, month, day, hour24, minute); if (frameTime > now) frameTime.setFullYear(year - 1); const diffMs = now - frameTime; minutes = Math.round(diffMs / (1000 * 60)); if (minutes < 0 || minutes > 120) minutes = null; } } catch(e) {} } return { index: index, minutesAgo: minutes }; }); }",
|
||||
"CheckModalOverlay": "() => { const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); if (bomOverlay && bomOverlay.style.display !== 'none') { return true; } const recaptchaSelectors = ['.g-recaptcha', '#g-recaptcha', '.rc-anchor-container']; for (const selector of recaptchaSelectors) { const el = document.querySelector(selector); if (el) { const style = window.getComputedStyle(el); const rect = el.getBoundingClientRect(); if (style.display !== 'none' && style.visibility !== 'hidden' && rect.width > 200 && rect.height > 200) { return true; } } } const allForms = document.querySelectorAll('form'); for (const form of allForms) { const action = form.getAttribute('action') || ''; const id = form.getAttribute('id') || ''; if (action.includes('feedback') || id.includes('feedback')) { const style = window.getComputedStyle(form); const rect = form.getBoundingClientRect(); if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { const text = form.textContent || ''; if (text.includes('reCAPTCHA') || text.includes('recaptcha') || text.includes('Tell us why')) { return true; } } } } return false; }",
|
||||
"CheckModalStillVisible": "() => { const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); if (bomOverlay && bomOverlay.style.display !== 'none') return true; const allForms = document.querySelectorAll('form'); for (const form of allForms) { const action = form.getAttribute('action') || ''; const id = form.getAttribute('id') || ''; if (action.includes('feedback') || id.includes('feedback')) { const style = window.getComputedStyle(form); const rect = form.getBoundingClientRect(); if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { return true; } } } return false; }",
|
||||
"GetViewportSize": "() => JSON.stringify({ width: window.innerWidth, height: window.innerHeight })",
|
||||
"ExtractWeatherMetadata": "() => { const section = document.querySelector('section[data-testid=\"weatherMetadata\"]') || document.querySelector('section[aria-label=\"Last updated\"]'); if (!section) return null; const divs = section.querySelectorAll('div'); return Array.from(divs).map(div => div.textContent.trim()).filter(text => text).join(' '); }"
|
||||
},
|
||||
"TextPatterns": {
|
||||
"ResultsCountPattern": "(\\d+)\\s+of\\s+(\\d+)",
|
||||
"TimestampPattern": "(?:[A-Za-z]+\\s+)?\\d{1,2}\\s+[A-Za-z]{3},?\\s+\\d{1,2}:\\d{2}\\s+(?:am|pm)",
|
||||
"ObservationTimePattern": "Observations:\\s*(\\d+)\\s*minutes?\\s*ago",
|
||||
"ForecastTimePattern": "Forecast:\\s*(\\d+)\\s*minutes?\\s+ago",
|
||||
"ForecastHourAgoPattern": "Forecast:\\s*an\\s+hour\\s+ago",
|
||||
"WeatherStationPattern": "at\\s+([^,]+)\\s+weather\\s+station",
|
||||
"DistancePattern": "(\\d+)\\s*km\\s+from",
|
||||
"ExpectedTexts": {
|
||||
"PlayButtonLabel": "Play",
|
||||
"PauseButtonLabel": "Pause",
|
||||
"RadarLinkText": "Rain radar and weather map"
|
||||
}
|
||||
},
|
||||
"Workflows": {
|
||||
"RadarScraping": {
|
||||
"Description": "Scrapes radar images for a location",
|
||||
"Steps": {
|
||||
"NavigateHomepage": { "Enabled": true },
|
||||
"ClickSearchButton": { "Enabled": true },
|
||||
"FillSearchInput": { "Enabled": true },
|
||||
"WaitForSearchResults": { "Enabled": true },
|
||||
"SelectSearchResult": { "Enabled": true },
|
||||
"ClickRadarLink": { "Enabled": true },
|
||||
"WaitForMapReady": { "Enabled": true },
|
||||
"PauseRadar": { "Enabled": true },
|
||||
"ResetToFirstFrame": { "Enabled": true },
|
||||
"ExtractMetadata": { "Enabled": true },
|
||||
"CalculateMapBounds": { "Enabled": true },
|
||||
"CaptureFrames": {
|
||||
"Enabled": true,
|
||||
"Parameters": {
|
||||
"FrameCount": 7,
|
||||
"WaitBetweenFrames": 5000
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"TemperatureMap": {
|
||||
"Description": "Scrapes temperature forecast map (future feature)",
|
||||
"Steps": {
|
||||
"NavigateHomepage": { "Enabled": true },
|
||||
"ClickSearchButton": { "Enabled": true },
|
||||
"FillSearchInput": { "Enabled": true },
|
||||
"WaitForSearchResults": { "Enabled": true },
|
||||
"SelectSearchResult": { "Enabled": true }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user