mirror of
https://github.com/alexhopeoconnor/bom-local-service.git
synced 2026-10-03 21:21:59 +10:00
alex
12dd0eedad
Major architectural changes: - Replace monolithic ScrapingService with workflow-based system - Extract scraping logic into discrete, testable steps - Add configuration-driven selectors, JavaScript templates, and text patterns - Implement step registry and workflow factory for extensibility - Add PageState tracking and step dependencies - Make IWorkflow generic to support different response types New components: - Scraping steps organized by category (Navigation, Search, Map, Metadata, Capture) - BaseScrapingStep base class with common dependencies - ScrapingContext for shared state between steps - RadarScrapingWorkflow orchestrating step execution - SelectorService for configurable element selection - Configuration models (SelectorConfig, TextPatternsConfig, JavaScriptTemplatesConfig) Performance and monitoring: - Add step-level timing metrics with historical tracking - Log step durations and compare to averages - Warn on slow steps/workflows (50%+ slower than average) - Integrate metrics with CacheService Configuration improvements: - Externalize all selectors to appsettings.json - Externalize JavaScript templates to appsettings.json - Externalize text patterns to appsettings.json - Support Docker configuration via mounted appsettings.json - Remove hardcoded waits, use configurable timeouts Bug fixes: - Fix DebugService collection modification exception - Fix ResetToFirstFrameStep click interception with JavaScript fallback - Remove minutes ago fallback, use timestamp parsing only - Update timestamp parsing for new BOM website format Documentation: - Update README with new architecture details - Add configuration section with Docker guidance - Add development guidelines for extending scraping system - Add performance monitoring documentation
65 lines
2.4 KiB
C#
65 lines
2.4 KiB
C#
using BomLocalService.Models;
|
|
using BomLocalService.Services.Interfaces;
|
|
using Microsoft.Playwright;
|
|
|
|
namespace BomLocalService.Services.Scraping.Steps;
|
|
|
|
/// <summary>
|
|
/// Base class for scraping steps with common functionality
|
|
/// </summary>
|
|
public abstract class BaseScrapingStep : IScrapingStep
|
|
{
|
|
protected readonly ILogger Logger;
|
|
protected readonly ISelectorService SelectorService;
|
|
protected readonly IDebugService DebugService;
|
|
protected readonly IConfiguration Configuration;
|
|
protected readonly ScrapingSelectorsConfig Selectors;
|
|
protected readonly JavaScriptTemplatesConfig JavaScriptTemplates;
|
|
protected readonly TextPatternsConfig TextPatterns;
|
|
|
|
protected BaseScrapingStep(
|
|
ILogger logger,
|
|
ISelectorService selectorService,
|
|
IDebugService debugService,
|
|
IConfiguration configuration)
|
|
{
|
|
Logger = logger;
|
|
SelectorService = selectorService;
|
|
DebugService = debugService;
|
|
Configuration = configuration;
|
|
|
|
Selectors = configuration.GetSection("Scraping:Selectors").Get<ScrapingSelectorsConfig>() ?? new();
|
|
JavaScriptTemplates = configuration.GetSection("Scraping:JavaScriptTemplates").Get<JavaScriptTemplatesConfig>() ?? new();
|
|
TextPatterns = configuration.GetSection("Scraping:TextPatterns").Get<TextPatternsConfig>() ?? new();
|
|
}
|
|
|
|
public abstract string Name { get; }
|
|
public abstract string[] Prerequisites { get; }
|
|
public abstract bool CanExecute(ScrapingContext context);
|
|
public abstract Task<ScrapingStepResult> ExecuteAsync(ScrapingContext context, CancellationToken cancellationToken);
|
|
|
|
protected async Task SaveDebugAsync(ScrapingContext context, int stepNumber, string stepName, CancellationToken cancellationToken)
|
|
{
|
|
await DebugService.SaveStepDebugAsync(
|
|
context.DebugFolder,
|
|
stepNumber,
|
|
stepName,
|
|
context.Page,
|
|
context.ConsoleMessages,
|
|
context.NetworkRequests,
|
|
cancellationToken);
|
|
}
|
|
|
|
protected async Task SaveErrorDebugAsync(ScrapingContext context, string errorMessage, CancellationToken cancellationToken)
|
|
{
|
|
await DebugService.SaveErrorDebugAsync(
|
|
context.DebugFolder,
|
|
errorMessage,
|
|
context.Page,
|
|
context.ConsoleMessages,
|
|
context.NetworkRequests,
|
|
cancellationToken);
|
|
}
|
|
}
|
|
|