mirror of
https://github.com/alexhopeoconnor/bom-local-service.git
synced 2026-10-03 21:21:59 +10:00
alex
12dd0eedad
Major architectural changes: - Replace monolithic ScrapingService with workflow-based system - Extract scraping logic into discrete, testable steps - Add configuration-driven selectors, JavaScript templates, and text patterns - Implement step registry and workflow factory for extensibility - Add PageState tracking and step dependencies - Make IWorkflow generic to support different response types New components: - Scraping steps organized by category (Navigation, Search, Map, Metadata, Capture) - BaseScrapingStep base class with common dependencies - ScrapingContext for shared state between steps - RadarScrapingWorkflow orchestrating step execution - SelectorService for configurable element selection - Configuration models (SelectorConfig, TextPatternsConfig, JavaScriptTemplatesConfig) Performance and monitoring: - Add step-level timing metrics with historical tracking - Log step durations and compare to averages - Warn on slow steps/workflows (50%+ slower than average) - Integrate metrics with CacheService Configuration improvements: - Externalize all selectors to appsettings.json - Externalize JavaScript templates to appsettings.json - Externalize text patterns to appsettings.json - Support Docker configuration via mounted appsettings.json - Remove hardcoded waits, use configurable timeouts Bug fixes: - Fix DebugService collection modification exception - Fix ResetToFirstFrameStep click interception with JavaScript fallback - Remove minutes ago fallback, use timestamp parsing only - Update timestamp parsing for new BOM website format Documentation: - Update README with new architecture details - Add configuration section with Docker guidance - Add development guidelines for extending scraping system - Add performance monitoring documentation
61 lines
1.8 KiB
C#
61 lines
1.8 KiB
C#
using BomLocalService.Models;
|
|
using BomLocalService.Services.Interfaces;
|
|
using Microsoft.Playwright;
|
|
|
|
namespace BomLocalService.Services;
|
|
|
|
public class SelectorService : ISelectorService
|
|
{
|
|
private readonly ILogger<SelectorService> _logger;
|
|
|
|
public SelectorService(ILogger<SelectorService> logger)
|
|
{
|
|
_logger = logger;
|
|
}
|
|
|
|
public async Task<ILocator?> FindElementAsync(IPage page, SelectorConfig config, CancellationToken cancellationToken = default)
|
|
{
|
|
foreach (var selector in config.Selectors)
|
|
{
|
|
try
|
|
{
|
|
var locator = page.Locator(selector).First;
|
|
if (await locator.IsVisibleAsync())
|
|
{
|
|
_logger.LogInformation("Found {Name} with selector: {Selector}", config.Name, selector);
|
|
return locator;
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogDebug(ex, "Selector {Selector} not found or not visible for {Name}", selector, config.Name);
|
|
continue;
|
|
}
|
|
}
|
|
|
|
if (config.Required)
|
|
{
|
|
var errorMsg = config.ErrorMessage ?? $"Required element {config.Name} not found with any selector";
|
|
_logger.LogWarning("{Message}", errorMsg);
|
|
}
|
|
else
|
|
{
|
|
_logger.LogDebug("Optional element {Name} not found with any selector", config.Name);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
public ILocator GetLocator(IPage page, SelectorConfig config)
|
|
{
|
|
if (config.Selectors.Length == 0)
|
|
{
|
|
throw new ArgumentException($"No selectors configured for {config.Name}");
|
|
}
|
|
|
|
// Returns first selector as locator (for cases where we know it exists)
|
|
return page.Locator(config.Selectors[0]).First;
|
|
}
|
|
}
|
|
|