mirror of
https://github.com/alexhopeoconnor/bom-local-service.git
synced 2026-10-04 03:28:11 +10:00
alex
12dd0eedad
Major architectural changes: - Replace monolithic ScrapingService with workflow-based system - Extract scraping logic into discrete, testable steps - Add configuration-driven selectors, JavaScript templates, and text patterns - Implement step registry and workflow factory for extensibility - Add PageState tracking and step dependencies - Make IWorkflow generic to support different response types New components: - Scraping steps organized by category (Navigation, Search, Map, Metadata, Capture) - BaseScrapingStep base class with common dependencies - ScrapingContext for shared state between steps - RadarScrapingWorkflow orchestrating step execution - SelectorService for configurable element selection - Configuration models (SelectorConfig, TextPatternsConfig, JavaScriptTemplatesConfig) Performance and monitoring: - Add step-level timing metrics with historical tracking - Log step durations and compare to averages - Warn on slow steps/workflows (50%+ slower than average) - Integrate metrics with CacheService Configuration improvements: - Externalize all selectors to appsettings.json - Externalize JavaScript templates to appsettings.json - Externalize text patterns to appsettings.json - Support Docker configuration via mounted appsettings.json - Remove hardcoded waits, use configurable timeouts Bug fixes: - Fix DebugService collection modification exception - Fix ResetToFirstFrameStep click interception with JavaScript fallback - Remove minutes ago fallback, use timestamp parsing only - Update timestamp parsing for new BOM website format Documentation: - Update README with new architecture details - Add configuration section with Docker guidance - Add development guidelines for extending scraping system - Add performance monitoring documentation
34 lines
954 B
C#
34 lines
954 B
C#
namespace BomLocalService.Models;
|
|
|
|
/// <summary>
|
|
/// Configuration for a single selector with multiple fallback options
|
|
/// </summary>
|
|
public class SelectorConfig
|
|
{
|
|
/// <summary>
|
|
/// Human-readable name for this selector (for logging/debugging)
|
|
/// </summary>
|
|
public string Name { get; set; } = string.Empty;
|
|
|
|
/// <summary>
|
|
/// Array of CSS selectors to try in order (first match wins)
|
|
/// </summary>
|
|
public string[] Selectors { get; set; } = Array.Empty<string>();
|
|
|
|
/// <summary>
|
|
/// Timeout in milliseconds when waiting for this element
|
|
/// </summary>
|
|
public int TimeoutMs { get; set; } = 5000;
|
|
|
|
/// <summary>
|
|
/// Whether this element is required (throws if not found)
|
|
/// </summary>
|
|
public bool Required { get; set; } = true;
|
|
|
|
/// <summary>
|
|
/// Custom error message if element is not found (when Required=true)
|
|
/// </summary>
|
|
public string? ErrorMessage { get; set; }
|
|
}
|
|
|