mirror of
https://github.com/alexhopeoconnor/bom-local-service.git
synced 2026-10-04 03:28:11 +10:00
alex
b30d76ddec
- Remove OpenAPI/Swagger support and all related code - Standardize configuration: remove hardcoded defaults, all defaults from appsettings.json - Configuration values throw exceptions if missing (except Debug:Enabled defaults to false) - Refactor CacheManagementService: extract ProcessLocationsAsync helper, remove duplicate code - Combine UpdateStaggerSeconds and LocationStaggerSeconds into single LocationStaggerSeconds config - Move config reads to constructors for better performance - Update README and docker-compose.yml to reflect configuration changes
246 lines
13 KiB
JSON
246 lines
13 KiB
JSON
{
|
|
"Logging": {
|
|
"LogLevel": {
|
|
"Default": "Information",
|
|
"Microsoft.AspNetCore": "Warning"
|
|
}
|
|
},
|
|
"AllowedHosts": "*",
|
|
"Cors": {
|
|
"AllowedOrigins": "*",
|
|
"AllowedMethods": "GET,POST,OPTIONS",
|
|
"AllowedHeaders": "*",
|
|
"AllowCredentials": false
|
|
},
|
|
"CacheDirectory": "./cache",
|
|
"CacheRetentionHours": 24,
|
|
"CacheExpirationMinutes": 12.5,
|
|
"CacheManagement": {
|
|
"CheckIntervalMinutes": 5,
|
|
"InitialDelaySeconds": 10,
|
|
"LocationStaggerSeconds": 1
|
|
},
|
|
"CacheCleanup": {
|
|
"IntervalHours": 1
|
|
},
|
|
"Screenshot": {
|
|
"DynamicContentWaitMs": 1500,
|
|
"TileRenderWaitMs": 3000,
|
|
"Crop": {
|
|
"X": 250,
|
|
"Y": 0,
|
|
"RightOffset": 250,
|
|
"Height": null
|
|
}
|
|
},
|
|
"Debug": {
|
|
"Enabled": false,
|
|
"WaitMs": 2000
|
|
},
|
|
"Timezone": "Australia/Brisbane",
|
|
"CachedDataTypes": {
|
|
"Radar": {
|
|
"FrameCount": 7
|
|
}
|
|
},
|
|
"TimeSeries": {
|
|
"WarningFolderCount": 200,
|
|
"MaxTimeRangeHours": null
|
|
},
|
|
"Scraping": {
|
|
"BaseUrl": "https://www.bom.gov.au/",
|
|
"Timeouts": {
|
|
"PageNavigation": 30000,
|
|
"ElementWait": 10000,
|
|
"SearchResults": 10000,
|
|
"MapRender": 15000,
|
|
"TileRender": 5000
|
|
},
|
|
"Selectors": {
|
|
"SearchButton": {
|
|
"Name": "Search Button",
|
|
"Selectors": [
|
|
"button[data-testid='searchLabel']",
|
|
"button[aria-label='Search for a location']",
|
|
"button.search-location__trigger-button",
|
|
"button.bom-button:has(span.bom-button__label:has-text('Search for a location'))",
|
|
"button:has-text('Search for a location')",
|
|
"[data-testid='showSearchModal'] button"
|
|
],
|
|
"TimeoutMs": 10000,
|
|
"Required": true,
|
|
"ErrorMessage": "Could not find 'Search for a location' button on BOM homepage."
|
|
},
|
|
"SearchInput": {
|
|
"Name": "Search Input",
|
|
"Selectors": ["#search-enter-keyword"],
|
|
"TimeoutMs": 5000,
|
|
"Required": true
|
|
},
|
|
"SearchResultsList": {
|
|
"Name": "Search Results List",
|
|
"Selectors": ["ul[aria-labelledby='location-results-title']"],
|
|
"TimeoutMs": 10000,
|
|
"Required": true
|
|
},
|
|
"SearchResultItem": {
|
|
"Name": "Search Result Item",
|
|
"Selectors": ["li.bom-linklist__item[role='listitem']"],
|
|
"TimeoutMs": 5000,
|
|
"Required": true
|
|
},
|
|
"LocationName": {
|
|
"Name": "Location Name",
|
|
"Selectors": ["[data-testid='location-name']"],
|
|
"TimeoutMs": 5000,
|
|
"Required": false
|
|
},
|
|
"LocationDescription": {
|
|
"Name": "Location Description",
|
|
"Selectors": [".bom-linklist-item__desc"],
|
|
"TimeoutMs": 5000,
|
|
"Required": false
|
|
},
|
|
"ResultsTitle": {
|
|
"Name": "Results Title",
|
|
"Selectors": ["#location-results-title", "[data-testid='location-results-title']"],
|
|
"TimeoutMs": 5000,
|
|
"Required": false
|
|
},
|
|
"RadarLink": {
|
|
"Name": "Radar Link",
|
|
"Selectors": [
|
|
"a.jump-link.bom-button.bom-button--secondary:has-text('Rain radar and weather map')",
|
|
".cta.button a.jump-link:has-text('Rain radar and weather map')",
|
|
"a.jump-link:has-text('Rain radar and weather map')",
|
|
".cta.button a:has-text('Rain radar and weather map')",
|
|
"a.bom-button--secondary:has-text('Rain radar and weather map')",
|
|
"a:has-text('Rain radar and weather map')",
|
|
"a:has-text('rain radar')",
|
|
"a[href*='radar']"
|
|
],
|
|
"TimeoutMs": 10000,
|
|
"Required": true,
|
|
"ErrorMessage": "Could not find 'Rain radar and weather map' link"
|
|
},
|
|
"MapCanvas": {
|
|
"Name": "Map Canvas",
|
|
"Selectors": [".esri-view-surface canvas"],
|
|
"TimeoutMs": 15000,
|
|
"Required": true
|
|
},
|
|
"MapContainer": {
|
|
"Name": "Map Container",
|
|
"Selectors": [".esri-view-surface"],
|
|
"TimeoutMs": 10000,
|
|
"Required": true
|
|
},
|
|
"PlayPauseButton": {
|
|
"Name": "Play Pause Button",
|
|
"Selectors": ["button[data-testid='bom-time-scrub-play-pause']"],
|
|
"TimeoutMs": 5000,
|
|
"Required": true
|
|
},
|
|
"PlayPauseLabel": {
|
|
"Name": "Play Pause Label",
|
|
"Selectors": [".bom-scrub-action__label"],
|
|
"TimeoutMs": 2000,
|
|
"Required": false
|
|
},
|
|
"FrameSegment": {
|
|
"Name": "Frame Segment",
|
|
"Selectors": ["[data-testid='bom-scrub-segment']"],
|
|
"TimeoutMs": 5000,
|
|
"Required": false
|
|
},
|
|
"StepForwardButton": {
|
|
"Name": "Step Forward Button",
|
|
"Selectors": ["button[data-testid='bom-scrub-utils__right__step-forward']"],
|
|
"TimeoutMs": 5000,
|
|
"Required": true
|
|
},
|
|
"TimeDisplayLabel": {
|
|
"Name": "Time Display Label",
|
|
"Selectors": [".bom-scrub-display-label"],
|
|
"TimeoutMs": 2000,
|
|
"Required": false
|
|
},
|
|
"ModalOverlay": {
|
|
"Name": "Modal Overlay",
|
|
"Selectors": [".bom-modal-overlay--after-open"],
|
|
"TimeoutMs": 1000,
|
|
"Required": false
|
|
},
|
|
"WeatherMetadata": {
|
|
"Name": "Weather Metadata",
|
|
"Selectors": ["section[data-testid='weatherMetadata']", "section[aria-label='Last updated']"],
|
|
"TimeoutMs": 5000,
|
|
"Required": false
|
|
}
|
|
},
|
|
"JavaScriptTemplates": {
|
|
"WaitForSearchResults": "() => { const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=\"listitem\"]')); return results.length > 0 && results.some(r => r.offsetParent !== null); }",
|
|
"ExtractSearchResults": "() => { const resultsList = document.querySelector('ul[aria-labelledby=\"location-results-title\"]'); if (!resultsList) { console.log('Location results list not found'); return []; } const results = Array.from(resultsList.querySelectorAll('li.bom-linklist__item[role=\"listitem\"]')); console.log('Found', results.length, 'location results'); return results.map((r) => { const nameEl = r.querySelector('[data-testid=\"location-name\"]'); const descEl = r.querySelector('.bom-linklist-item__desc'); const name = nameEl ? (nameEl.textContent || nameEl.innerText || '').trim() : ''; const desc = descEl ? (descEl.textContent || descEl.innerText || '').trim() : ''; const fullText = (r.textContent || r.innerText || '').trim(); console.log('Result:', { hasNameEl: !!nameEl, hasDescEl: !!descEl, name: name, desc: desc }); return [name, desc, fullText]; }); }",
|
|
"ExtractSearchResultsFallback": "() => { const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=\"listitem\"]')); return results.map(r => r.textContent || ''); }",
|
|
"WaitForMapCanvas": "() => { const canvas = document.querySelector('.esri-view-surface canvas'); return canvas && canvas.width > 0 && canvas.height > 0 && canvas.offsetWidth > 0 && canvas.offsetHeight > 0; }",
|
|
"WaitForEsriView": "() => { try { const elements = document.querySelectorAll('.esri-view'); for (let el of elements) { if (el.__view && el.__view.ready) { return true; } } } catch(e) {} return false; }",
|
|
"CheckActiveFrameSegment": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); const activeSegment = segments.find(s => { const style = window.getComputedStyle(s); return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)'; }); return activeSegment && activeSegment.getAttribute('data-id') === '0'; }",
|
|
"WaitForMapContainer": "() => { const container = document.querySelector('.esri-view-surface'); return container && container.offsetWidth > 0 && container.offsetHeight > 0; }",
|
|
"ExtractFrameInfo": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); const now = new Date(); return segments.map((seg, index) => { const ariaLabel = seg.getAttribute('aria-label') || ''; let minutes = null; const timestampMatch = ariaLabel.match(/(?:[A-Za-z]+\\s+)?(\\d{1,2})\\s+([A-Za-z]{3}),?\\s+(\\d{1,2}):(\\d{2})\\s+(am|pm)/i); if (timestampMatch) { try { const day = parseInt(timestampMatch[1]); const monthStr = timestampMatch[2]; const hour12 = parseInt(timestampMatch[3]); const minute = parseInt(timestampMatch[4]); const ampm = timestampMatch[5].toLowerCase(); const monthNames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']; const month = monthNames.indexOf(monthStr.toLowerCase()); if (month >= 0) { let hour24 = hour12; if (ampm === 'pm' && hour12 !== 12) hour24 += 12; if (ampm === 'am' && hour12 === 12) hour24 = 0; const year = now.getFullYear(); const frameTime = new Date(year, month, day, hour24, minute); if (frameTime > now) frameTime.setFullYear(year - 1); const diffMs = now - frameTime; minutes = Math.round(diffMs / (1000 * 60)); if (minutes < 0 || minutes > 120) minutes = null; } } catch(e) {} } return { index: index, minutesAgo: minutes }; }); }",
|
|
"CheckModalOverlay": "() => { const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); if (bomOverlay && bomOverlay.style.display !== 'none') { return true; } const recaptchaSelectors = ['.g-recaptcha', '#g-recaptcha', '.rc-anchor-container']; for (const selector of recaptchaSelectors) { const el = document.querySelector(selector); if (el) { const style = window.getComputedStyle(el); const rect = el.getBoundingClientRect(); if (style.display !== 'none' && style.visibility !== 'hidden' && rect.width > 200 && rect.height > 200) { return true; } } } const allForms = document.querySelectorAll('form'); for (const form of allForms) { const action = form.getAttribute('action') || ''; const id = form.getAttribute('id') || ''; if (action.includes('feedback') || id.includes('feedback')) { const style = window.getComputedStyle(form); const rect = form.getBoundingClientRect(); if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { const text = form.textContent || ''; if (text.includes('reCAPTCHA') || text.includes('recaptcha') || text.includes('Tell us why')) { return true; } } } } return false; }",
|
|
"CheckModalStillVisible": "() => { const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); if (bomOverlay && bomOverlay.style.display !== 'none') return true; const allForms = document.querySelectorAll('form'); for (const form of allForms) { const action = form.getAttribute('action') || ''; const id = form.getAttribute('id') || ''; if (action.includes('feedback') || id.includes('feedback')) { const style = window.getComputedStyle(form); const rect = form.getBoundingClientRect(); if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { return true; } } } return false; }",
|
|
"GetViewportSize": "() => JSON.stringify({ width: window.innerWidth, height: window.innerHeight })",
|
|
"ExtractWeatherMetadata": "() => { const section = document.querySelector('section[data-testid=\"weatherMetadata\"]') || document.querySelector('section[aria-label=\"Last updated\"]'); if (!section) return null; const divs = section.querySelectorAll('div'); return Array.from(divs).map(div => div.textContent.trim()).filter(text => text).join(' '); }"
|
|
},
|
|
"TextPatterns": {
|
|
"ResultsCountPattern": "(\\d+)\\s+of\\s+(\\d+)",
|
|
"TimestampPattern": "(?:[A-Za-z]+\\s+)?\\d{1,2}\\s+[A-Za-z]{3},?\\s+\\d{1,2}:\\d{2}\\s+(?:am|pm)",
|
|
"ObservationTimePattern": "Observations:\\s*(\\d+)\\s*minutes?\\s*ago",
|
|
"ForecastTimePattern": "Forecast:\\s*(\\d+)\\s*minutes?\\s+ago",
|
|
"ForecastHourAgoPattern": "Forecast:\\s*an\\s+hour\\s+ago",
|
|
"WeatherStationPattern": "at\\s+([^,]+)\\s+weather\\s+station",
|
|
"DistancePattern": "(\\d+)\\s*km\\s+from",
|
|
"ExpectedTexts": {
|
|
"PlayButtonLabel": "Play",
|
|
"PauseButtonLabel": "Pause",
|
|
"RadarLinkText": "Rain radar and weather map"
|
|
}
|
|
},
|
|
"Workflows": {
|
|
"RadarScraping": {
|
|
"Description": "Scrapes radar images for a location",
|
|
"Steps": {
|
|
"NavigateHomepage": { "Enabled": true },
|
|
"ClickSearchButton": { "Enabled": true },
|
|
"FillSearchInput": { "Enabled": true },
|
|
"WaitForSearchResults": { "Enabled": true },
|
|
"SelectSearchResult": { "Enabled": true },
|
|
"ClickRadarLink": { "Enabled": true },
|
|
"WaitForMapReady": { "Enabled": true },
|
|
"PauseRadar": { "Enabled": true },
|
|
"ResetToFirstFrame": { "Enabled": true },
|
|
"ExtractMetadata": { "Enabled": true },
|
|
"CalculateMapBounds": { "Enabled": true },
|
|
"CaptureFrames": {
|
|
"Enabled": true,
|
|
"Parameters": {
|
|
"FrameCount": 7,
|
|
"WaitBetweenFrames": 5000
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"TemperatureMap": {
|
|
"Description": "Scrapes temperature forecast map (future feature)",
|
|
"Steps": {
|
|
"NavigateHomepage": { "Enabled": true },
|
|
"ClickSearchButton": { "Enabled": true },
|
|
"FillSearchInput": { "Enabled": true },
|
|
"WaitForSearchResults": { "Enabled": true },
|
|
"SelectSearchResult": { "Enabled": true }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|