Files
bom-local-service/Services/DebugService.cs
T
alex 12dd0eedad Refactor scraping system to workflow-based architecture
Major architectural changes:
- Replace monolithic ScrapingService with workflow-based system
- Extract scraping logic into discrete, testable steps
- Add configuration-driven selectors, JavaScript templates, and text patterns
- Implement step registry and workflow factory for extensibility
- Add PageState tracking and step dependencies
- Make IWorkflow generic to support different response types

New components:
- Scraping steps organized by category (Navigation, Search, Map, Metadata, Capture)
- BaseScrapingStep base class with common dependencies
- ScrapingContext for shared state between steps
- RadarScrapingWorkflow orchestrating step execution
- SelectorService for configurable element selection
- Configuration models (SelectorConfig, TextPatternsConfig, JavaScriptTemplatesConfig)

Performance and monitoring:
- Add step-level timing metrics with historical tracking
- Log step durations and compare to averages
- Warn on slow steps/workflows (50%+ slower than average)
- Integrate metrics with CacheService

Configuration improvements:
- Externalize all selectors to appsettings.json
- Externalize JavaScript templates to appsettings.json
- Externalize text patterns to appsettings.json
- Support Docker configuration via mounted appsettings.json
- Remove hardcoded waits, use configurable timeouts

Bug fixes:
- Fix DebugService collection modification exception
- Fix ResetToFirstFrameStep click interception with JavaScript fallback
- Remove minutes ago fallback, use timestamp parsing only
- Update timestamp parsing for new BOM website format

Documentation:
- Update README with new architecture details
- Add configuration section with Docker guidance
- Add development guidelines for extending scraping system
- Add performance monitoring documentation
2025-12-18 00:54:57 +10:00

185 lines
8.4 KiB
C#

using BomLocalService.Services.Interfaces;
using BomLocalService.Utilities;
using Microsoft.Playwright;
namespace BomLocalService.Services;
public class DebugService : IDebugService
{
private readonly ILogger<DebugService> _logger;
private readonly bool _enabled;
private readonly string _debugDirectory;
private readonly int _waitMs;
public DebugService(ILogger<DebugService> logger, IConfiguration configuration)
{
_logger = logger;
_waitMs = configuration.GetValue<int>("Debug:WaitMs", 2000);
// Check all possible ways the config might be set
var debugEnabledEnv = Environment.GetEnvironmentVariable("DEBUG__ENABLED");
var debugEnabledConfig = configuration.GetValue<bool>("Debug:Enabled", false);
var debugEnabledFromConfig = configuration["Debug:Enabled"];
_logger.LogInformation("Debug configuration check - Env var DEBUG__ENABLED: {EnvVar}, Config Debug:Enabled: {ConfigValue}, Config string: {ConfigString}",
debugEnabledEnv ?? "null", debugEnabledConfig, debugEnabledFromConfig ?? "null");
// Try environment variable first, then config
_enabled = !string.IsNullOrEmpty(debugEnabledEnv)
? bool.TryParse(debugEnabledEnv, out var envBool) && envBool
: debugEnabledConfig;
var cacheDirectory = FilePathHelper.GetCacheDirectory(configuration);
_debugDirectory = Path.Combine(cacheDirectory, "debug");
if (_enabled)
{
Directory.CreateDirectory(_debugDirectory);
_logger.LogInformation("Debug mode is ENABLED. Debug files will be saved to: {DebugDirectory}", _debugDirectory);
}
else
{
_logger.LogInformation("Debug mode is DISABLED");
}
}
public bool IsEnabled => _enabled;
public string CreateRequestFolder(string requestId)
{
if (!_enabled) return string.Empty;
var requestFolder = Path.Combine(_debugDirectory, requestId);
Directory.CreateDirectory(requestFolder);
_logger.LogDebug("Created debug folder for request: {RequestId} at {Path}", requestId, requestFolder);
return requestFolder;
}
public async Task SaveStepDebugAsync(string requestFolder, int stepNumber, string stepName, IPage page, List<(string type, string text, DateTime timestamp)>? consoleMessages = null, List<(string method, string url, int? status, string resourceType, DateTime timestamp)>? networkRequests = null, CancellationToken cancellationToken = default)
{
if (!_enabled || string.IsNullOrEmpty(requestFolder)) return;
try
{
// Wait for page to be fully loaded before taking screenshot
try
{
await page.WaitForLoadStateAsync(LoadState.NetworkIdle, new PageWaitForLoadStateOptions { Timeout = 10000 });
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 5000 });
}
catch
{
// If network idle times out, at least wait for DOM
}
await page.WaitForTimeoutAsync(_waitMs); // Additional wait for dynamic content and animations
var stepFolder = Path.Combine(requestFolder, $"step_{stepNumber:D2}_{stepName}");
Directory.CreateDirectory(stepFolder);
// Save screenshot
var screenshotPath = Path.Combine(stepFolder, "screenshot.png");
await page.ScreenshotAsync(new PageScreenshotOptions
{
Path = screenshotPath,
FullPage = true
});
// Save HTML
var htmlContent = await page.ContentAsync();
var htmlPath = Path.Combine(stepFolder, "page.html");
await File.WriteAllTextAsync(htmlPath, htmlContent, cancellationToken);
// Save console messages if provided
// Create a snapshot to avoid collection modification during enumeration
if (consoleMessages != null && consoleMessages.Count > 0)
{
var consolePath = Path.Combine(stepFolder, "console.log");
var consoleSnapshot = consoleMessages.ToList(); // Create snapshot
var consoleText = string.Join("\n", consoleSnapshot.Select(msg => $"[{msg.timestamp:HH:mm:ss.fff}] [{msg.type}] {msg.text}"));
await File.WriteAllTextAsync(consolePath, consoleText, cancellationToken);
}
// Save network requests summary if provided
// Create a snapshot to avoid collection modification during enumeration
if (networkRequests != null && networkRequests.Count > 0)
{
var networkPath = Path.Combine(stepFolder, "network.log");
var networkSnapshot = networkRequests.ToList(); // Create snapshot
var networkText = string.Join("\n", networkSnapshot.Select(req =>
$"[{req.timestamp:HH:mm:ss.fff}] {req.method} {req.url} -> {req.status?.ToString() ?? "pending"} ({req.resourceType})"
));
await File.WriteAllTextAsync(networkPath, networkText, cancellationToken);
}
_logger.LogDebug("Saved debug files for step {StepNumber} ({StepName}) to {Path}", stepNumber, stepName, stepFolder);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to save debug files for step {StepNumber} ({StepName})", stepNumber, stepName);
}
}
public async Task SaveErrorDebugAsync(string requestFolder, string errorMessage, IPage? page = null,
List<(string type, string text, DateTime timestamp)>? consoleMessages = null,
List<(string method, string url, int? status, string resourceType, DateTime timestamp)>? networkRequests = null,
CancellationToken cancellationToken = default)
{
if (!_enabled || string.IsNullOrEmpty(requestFolder)) return;
try
{
var errorFolder = Path.Combine(requestFolder, "error");
Directory.CreateDirectory(errorFolder);
// Save error message
var errorPath = Path.Combine(errorFolder, "error.txt");
await File.WriteAllTextAsync(errorPath, errorMessage, cancellationToken);
// Save screenshot if page is available
if (page != null)
{
var screenshotPath = Path.Combine(errorFolder, "screenshot.png");
await page.ScreenshotAsync(new PageScreenshotOptions
{
Path = screenshotPath,
FullPage = true
});
// Save HTML
var htmlContent = await page.ContentAsync();
var htmlPath = Path.Combine(errorFolder, "page.html");
await File.WriteAllTextAsync(htmlPath, htmlContent, cancellationToken);
}
// Save console messages if provided
// Create a snapshot to avoid collection modification during enumeration
if (consoleMessages != null && consoleMessages.Count > 0)
{
var consolePath = Path.Combine(errorFolder, "console.log");
var consoleSnapshot = consoleMessages.ToList(); // Create snapshot
var consoleText = string.Join("\n", consoleSnapshot.Select(msg => $"[{msg.timestamp:HH:mm:ss.fff}] [{msg.type}] {msg.text}"));
await File.WriteAllTextAsync(consolePath, consoleText, cancellationToken);
}
// Save network requests summary if provided
// Create a snapshot to avoid collection modification during enumeration
if (networkRequests != null && networkRequests.Count > 0)
{
var networkPath = Path.Combine(errorFolder, "network.log");
var networkSnapshot = networkRequests.ToList(); // Create snapshot
var networkText = string.Join("\n", networkSnapshot.Select(req =>
$"[{req.timestamp:HH:mm:ss.fff}] {req.method} {req.url} -> {req.status?.ToString() ?? "pending"} ({req.resourceType})"
));
await File.WriteAllTextAsync(networkPath, networkText, cancellationToken);
}
_logger.LogDebug("Saved error debug files to {Path}", errorFolder);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to save error debug files");
}
}
}