mirror of
https://github.com/alexhopeoconnor/bom-local-service.git
synced 2026-10-04 03:28:11 +10:00
alex
12dd0eedad
Major architectural changes: - Replace monolithic ScrapingService with workflow-based system - Extract scraping logic into discrete, testable steps - Add configuration-driven selectors, JavaScript templates, and text patterns - Implement step registry and workflow factory for extensibility - Add PageState tracking and step dependencies - Make IWorkflow generic to support different response types New components: - Scraping steps organized by category (Navigation, Search, Map, Metadata, Capture) - BaseScrapingStep base class with common dependencies - ScrapingContext for shared state between steps - RadarScrapingWorkflow orchestrating step execution - SelectorService for configurable element selection - Configuration models (SelectorConfig, TextPatternsConfig, JavaScriptTemplatesConfig) Performance and monitoring: - Add step-level timing metrics with historical tracking - Log step durations and compare to averages - Warn on slow steps/workflows (50%+ slower than average) - Integrate metrics with CacheService Configuration improvements: - Externalize all selectors to appsettings.json - Externalize JavaScript templates to appsettings.json - Externalize text patterns to appsettings.json - Support Docker configuration via mounted appsettings.json - Remove hardcoded waits, use configurable timeouts Bug fixes: - Fix DebugService collection modification exception - Fix ResetToFirstFrameStep click interception with JavaScript fallback - Remove minutes ago fallback, use timestamp parsing only - Update timestamp parsing for new BOM website format Documentation: - Update README with new architecture details - Add configuration section with Docker guidance - Add development guidelines for extending scraping system - Add performance monitoring documentation
53 lines
1.8 KiB
C#
53 lines
1.8 KiB
C#
using BomLocalService.Models;
|
|
using BomLocalService.Services.Interfaces;
|
|
using BomLocalService.Services.Scraping;
|
|
using Microsoft.Playwright;
|
|
|
|
namespace BomLocalService.Services;
|
|
|
|
public class ScrapingService : IScrapingService
|
|
{
|
|
private readonly ILogger<ScrapingService> _logger;
|
|
private readonly IWorkflowFactory _workflowFactory;
|
|
|
|
public ScrapingService(
|
|
ILogger<ScrapingService> logger,
|
|
IWorkflowFactory workflowFactory)
|
|
{
|
|
_logger = logger;
|
|
_workflowFactory = workflowFactory;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Scrapes the BOM website to get a radar screenshot for a location
|
|
/// </summary>
|
|
public async Task<RadarResponse> ScrapeRadarScreenshotAsync(
|
|
string suburb,
|
|
string state,
|
|
string cacheFolderPath,
|
|
string debugFolder,
|
|
IPage page,
|
|
List<(string type, string text, DateTime timestamp)> consoleMessages,
|
|
List<(string method, string url, int? status, string resourceType, DateTime timestamp)> networkRequests,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var context = new ScrapingContext
|
|
{
|
|
Page = page,
|
|
Suburb = suburb,
|
|
State = state,
|
|
CacheFolderPath = cacheFolderPath,
|
|
DebugFolder = debugFolder,
|
|
ConsoleMessages = consoleMessages,
|
|
NetworkRequests = networkRequests
|
|
};
|
|
|
|
// Get the appropriate workflow (currently only RadarScraping is implemented)
|
|
var workflow = _workflowFactory.GetWorkflow<RadarResponse>("RadarScraping");
|
|
|
|
_logger.LogInformation("Executing {Workflow} workflow for {Suburb}, {State}", workflow.Name, suburb, state);
|
|
|
|
return await workflow.ExecuteAsync(context, cancellationToken);
|
|
}
|
|
}
|