mirror of
https://github.com/alexhopeoconnor/bom-local-service.git
synced 2026-10-04 03:28:11 +10:00
Major architectural changes: - Replace monolithic ScrapingService with workflow-based system - Extract scraping logic into discrete, testable steps - Add configuration-driven selectors, JavaScript templates, and text patterns - Implement step registry and workflow factory for extensibility - Add PageState tracking and step dependencies - Make IWorkflow generic to support different response types New components: - Scraping steps organized by category (Navigation, Search, Map, Metadata, Capture) - BaseScrapingStep base class with common dependencies - ScrapingContext for shared state between steps - RadarScrapingWorkflow orchestrating step execution - SelectorService for configurable element selection - Configuration models (SelectorConfig, TextPatternsConfig, JavaScriptTemplatesConfig) Performance and monitoring: - Add step-level timing metrics with historical tracking - Log step durations and compare to averages - Warn on slow steps/workflows (50%+ slower than average) - Integrate metrics with CacheService Configuration improvements: - Externalize all selectors to appsettings.json - Externalize JavaScript templates to appsettings.json - Externalize text patterns to appsettings.json - Support Docker configuration via mounted appsettings.json - Remove hardcoded waits, use configurable timeouts Bug fixes: - Fix DebugService collection modification exception - Fix ResetToFirstFrameStep click interception with JavaScript fallback - Remove minutes ago fallback, use timestamp parsing only - Update timestamp parsing for new BOM website format Documentation: - Update README with new architecture details - Add configuration section with Docker guidance - Add development guidelines for extending scraping system - Add performance monitoring documentation
187 lines
8.4 KiB
C#
187 lines
8.4 KiB
C#
namespace BomLocalService.Models;
|
|
|
|
/// <summary>
|
|
/// JavaScript code templates used for page evaluation
|
|
/// </summary>
|
|
public class JavaScriptTemplatesConfig
|
|
{
|
|
public string WaitForSearchResults { get; set; } = @"() => {
|
|
const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=""listitem""]'));
|
|
return results.length > 0 && results.some(r => r.offsetParent !== null);
|
|
}";
|
|
|
|
public string ExtractSearchResults { get; set; } = @"() => {
|
|
const resultsList = document.querySelector('ul[aria-labelledby=""location-results-title""]');
|
|
if (!resultsList) {
|
|
console.log('Location results list not found');
|
|
return [];
|
|
}
|
|
const results = Array.from(resultsList.querySelectorAll('li.bom-linklist__item[role=""listitem""]'));
|
|
console.log('Found', results.length, 'location results');
|
|
return results.map((r) => {
|
|
const nameEl = r.querySelector('[data-testid=""location-name""]');
|
|
const descEl = r.querySelector('.bom-linklist-item__desc');
|
|
const name = nameEl ? (nameEl.textContent || nameEl.innerText || '').trim() : '';
|
|
const desc = descEl ? (descEl.textContent || descEl.innerText || '').trim() : '';
|
|
const fullText = (r.textContent || r.innerText || '').trim();
|
|
console.log('Result:', { hasNameEl: !!nameEl, hasDescEl: !!descEl, name: name, desc: desc });
|
|
return [name, desc, fullText];
|
|
});
|
|
}";
|
|
|
|
public string ExtractSearchResultsFallback { get; set; } = @"() => {
|
|
const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=""listitem""]'));
|
|
return results.map(r => r.textContent || '');
|
|
}";
|
|
|
|
public string WaitForMapCanvas { get; set; } = @"() => {
|
|
const canvas = document.querySelector('.esri-view-surface canvas');
|
|
return canvas && canvas.width > 0 && canvas.height > 0 && canvas.offsetWidth > 0 && canvas.offsetHeight > 0;
|
|
}";
|
|
|
|
public string WaitForEsriView { get; set; } = @"() => {
|
|
try {
|
|
const elements = document.querySelectorAll('.esri-view');
|
|
for (let el of elements) {
|
|
if (el.__view && el.__view.ready) {
|
|
return true;
|
|
}
|
|
}
|
|
} catch(e) {}
|
|
return false;
|
|
}";
|
|
|
|
public string CheckActiveFrameSegment { get; set; } = @"() => {
|
|
const segments = Array.from(document.querySelectorAll('[data-testid=""bom-scrub-segment""]'));
|
|
const activeSegment = segments.find(s => {
|
|
const style = window.getComputedStyle(s);
|
|
return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)';
|
|
});
|
|
return activeSegment && activeSegment.getAttribute('data-id') === '0';
|
|
}";
|
|
|
|
public string WaitForMapContainer { get; set; } = @"() => {
|
|
const container = document.querySelector('.esri-view-surface');
|
|
return container && container.offsetWidth > 0 && container.offsetHeight > 0;
|
|
}";
|
|
|
|
public string ExtractFrameInfo { get; set; } = @"() => {
|
|
const segments = Array.from(document.querySelectorAll('[data-testid=""bom-scrub-segment""]'));
|
|
const now = new Date();
|
|
return segments.map((seg, index) => {
|
|
const ariaLabel = seg.getAttribute('aria-label') || '';
|
|
let minutes = null;
|
|
|
|
// Parse timestamp format: ""Wednesday 17 Dec, 11:05 pm"" or ""17 Dec, 11:05 pm""
|
|
const timestampMatch = ariaLabel.match(/(?:[A-Za-z]+\s+)?(\d{1,2})\s+([A-Za-z]{3}),?\s+(\d{1,2}):(\d{2})\s+(am|pm)/i);
|
|
if (timestampMatch) {
|
|
try {
|
|
const day = parseInt(timestampMatch[1]);
|
|
const monthStr = timestampMatch[2];
|
|
const hour12 = parseInt(timestampMatch[3]);
|
|
const minute = parseInt(timestampMatch[4]);
|
|
const ampm = timestampMatch[5].toLowerCase();
|
|
|
|
const monthNames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];
|
|
const month = monthNames.indexOf(monthStr.toLowerCase());
|
|
|
|
if (month >= 0) {
|
|
let hour24 = hour12;
|
|
if (ampm === 'pm' && hour12 !== 12) hour24 += 12;
|
|
if (ampm === 'am' && hour12 === 12) hour24 = 0;
|
|
|
|
const year = now.getFullYear();
|
|
const frameTime = new Date(year, month, day, hour24, minute);
|
|
|
|
// If parsed time is in future, assume it's from last year
|
|
if (frameTime > now) {
|
|
frameTime.setFullYear(year - 1);
|
|
}
|
|
|
|
const diffMs = now - frameTime;
|
|
minutes = Math.round(diffMs / (1000 * 60));
|
|
|
|
// Validate reasonable range (0-2 hours)
|
|
if (minutes < 0 || minutes > 120) {
|
|
minutes = null;
|
|
}
|
|
}
|
|
} catch(e) {
|
|
// Parse error, leave minutes as null
|
|
}
|
|
}
|
|
|
|
return { index: index, minutesAgo: minutes };
|
|
});
|
|
}";
|
|
|
|
public string CheckModalOverlay { get; set; } = @"() => {
|
|
const bomOverlay = document.querySelector('.bom-modal-overlay--after-open');
|
|
if (bomOverlay && bomOverlay.style.display !== 'none') {
|
|
return true;
|
|
}
|
|
|
|
const recaptchaSelectors = ['.g-recaptcha', '#g-recaptcha', '.rc-anchor-container'];
|
|
for (const selector of recaptchaSelectors) {
|
|
const el = document.querySelector(selector);
|
|
if (el) {
|
|
const style = window.getComputedStyle(el);
|
|
const rect = el.getBoundingClientRect();
|
|
if (style.display !== 'none' && style.visibility !== 'hidden' &&
|
|
rect.width > 200 && rect.height > 200) {
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
const allForms = document.querySelectorAll('form');
|
|
for (const form of allForms) {
|
|
const action = form.getAttribute('action') || '';
|
|
const id = form.getAttribute('id') || '';
|
|
if (action.includes('feedback') || id.includes('feedback')) {
|
|
const style = window.getComputedStyle(form);
|
|
const rect = form.getBoundingClientRect();
|
|
if (style.display !== 'none' && rect.width > 200 && rect.height > 200) {
|
|
const text = form.textContent || '';
|
|
if (text.includes('reCAPTCHA') || text.includes('recaptcha') || text.includes('Tell us why')) {
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}";
|
|
|
|
public string CheckModalStillVisible { get; set; } = @"() => {
|
|
const bomOverlay = document.querySelector('.bom-modal-overlay--after-open');
|
|
if (bomOverlay && bomOverlay.style.display !== 'none') return true;
|
|
|
|
const allForms = document.querySelectorAll('form');
|
|
for (const form of allForms) {
|
|
const action = form.getAttribute('action') || '';
|
|
const id = form.getAttribute('id') || '';
|
|
if (action.includes('feedback') || id.includes('feedback')) {
|
|
const style = window.getComputedStyle(form);
|
|
const rect = form.getBoundingClientRect();
|
|
if (style.display !== 'none' && rect.width > 200 && rect.height > 200) {
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
return false;
|
|
}";
|
|
|
|
public string GetViewportSize { get; set; } = @"() => JSON.stringify({ width: window.innerWidth, height: window.innerHeight })";
|
|
|
|
public string ExtractWeatherMetadata { get; set; } = @"() => {
|
|
const section = document.querySelector('section[data-testid=""weatherMetadata""]') ||
|
|
document.querySelector('section[aria-label=""Last updated""]');
|
|
if (!section) return null;
|
|
|
|
const divs = section.querySelectorAll('div');
|
|
return Array.from(divs).map(div => div.textContent.trim()).filter(text => text).join(' ');
|
|
}";
|
|
}
|
|
|