{ "Logging": { "LogLevel": { "Default": "Information", "Microsoft.AspNetCore": "Warning" } }, "AllowedHosts": "*", "Cors": { "AllowedOrigins": "*", "AllowedMethods": "GET,POST,OPTIONS", "AllowedHeaders": "*", "AllowCredentials": false }, "CacheDirectory": "./cache", "CacheRetentionHours": 24, "CacheExpirationMinutes": 12.5, "CacheManagement": { "CheckIntervalMinutes": 5, "InitialDelaySeconds": 10, "LocationStaggerSeconds": 1 }, "CacheCleanup": { "IntervalHours": 1 }, "Screenshot": { "DynamicContentWaitMs": 1500, "TileRenderWaitMs": 3000, "Crop": { "X": 250, "Y": 0, "RightOffset": 250, "Height": null } }, "Debug": { "Enabled": false, "WaitMs": 2000 }, "Timezone": "Australia/Brisbane", "CachedDataTypes": { "Radar": { "FrameCount": 7 } }, "TimeSeries": { "WarningFolderCount": 200, "MaxTimeRangeHours": null }, "Scraping": { "BaseUrl": "https://www.bom.gov.au/", "Timeouts": { "PageNavigation": 30000, "ElementWait": 10000, "SearchResults": 10000, "MapRender": 15000, "TileRender": 5000 }, "Selectors": { "SearchButton": { "Name": "Search Button", "Selectors": [ "button[data-testid='searchLabel']", "button.search-location__trigger-button", "button[aria-label='Search for a location'].search-location__trigger-button", "button.search-location__trigger-button.bom-button:has(span.bom-button__label:has-text('Search for a location'))" ], "TimeoutMs": 10000, "Required": true, "ErrorMessage": "Could not find 'Search for a location' button on BOM homepage." }, "SearchInput": { "Name": "Search Input", "Selectors": ["#search-enter-keyword"], "TimeoutMs": 5000, "Required": true }, "SearchResultsList": { "Name": "Search Results List", "Selectors": ["ul[aria-labelledby='location-results-title']"], "TimeoutMs": 10000, "Required": true }, "SearchResultItem": { "Name": "Search Result Item", "Selectors": ["li.bom-linklist__item[role='listitem']"], "TimeoutMs": 5000, "Required": true }, "LocationName": { "Name": "Location Name", "Selectors": ["[data-testid='location-name']"], "TimeoutMs": 5000, "Required": false }, "LocationDescription": { "Name": "Location Description", "Selectors": [".bom-linklist-item__desc"], "TimeoutMs": 5000, "Required": false }, "ResultsTitle": { "Name": "Results Title", "Selectors": ["#location-results-title", "[data-testid='location-results-title']"], "TimeoutMs": 5000, "Required": false }, "RadarLink": { "Name": "Radar Link", "Selectors": [ "a.jump-link.bom-button.bom-button--secondary:has-text('Rain radar and weather map')", ".cta.button a.jump-link:has-text('Rain radar and weather map')", "a.jump-link:has-text('Rain radar and weather map')", ".cta.button a:has-text('Rain radar and weather map')", "a.bom-button--secondary:has-text('Rain radar and weather map')", "a:has-text('Rain radar and weather map')", "a:has-text('rain radar')", "a[href*='radar']" ], "TimeoutMs": 10000, "Required": true, "ErrorMessage": "Could not find 'Rain radar and weather map' link" }, "MapCanvas": { "Name": "Map Canvas", "Selectors": [".esri-view-surface canvas"], "TimeoutMs": 15000, "Required": true }, "MapContainer": { "Name": "Map Container", "Selectors": [".esri-view-surface"], "TimeoutMs": 10000, "Required": true }, "PlayPauseButton": { "Name": "Play Pause Button", "Selectors": [ "button[data-testid='bom-time-scrub-step-through']", "button[data-testid='bom-time-scrub-play-pause']" ], "TimeoutMs": 5000, "Required": true }, "PlayPauseLabel": { "Name": "Play Pause Label", "Selectors": [".bom-scrub-action__label"], "TimeoutMs": 2000, "Required": false }, "FrameSegment": { "Name": "Frame Segment", "Selectors": ["[data-testid='bom-scrub-segment']"], "TimeoutMs": 5000, "Required": false }, "StepForwardButton": { "Name": "Step Forward Button", "Selectors": [ "button[data-testid='bom-time-scrub-step-through']", "button.bom-scrub-action--step-through", "button[data-testid='bom-scrub-utils__right__step-forward']" ], "TimeoutMs": 5000, "Required": true }, "TimeDisplayLabel": { "Name": "Time Display Label", "Selectors": [".bom-scrub-display-label"], "TimeoutMs": 2000, "Required": false }, "ModalOverlay": { "Name": "Modal Overlay", "Selectors": [".bom-modal-overlay--after-open"], "TimeoutMs": 1000, "Required": false }, "WeatherMetadata": { "Name": "Weather Metadata", "Selectors": ["section[data-testid='weatherMetadata']", "section[aria-label='Last updated']"], "TimeoutMs": 5000, "Required": false } }, "JavaScriptTemplates": { "WaitForSearchResults": "() => { const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=\"listitem\"]')); return results.length > 0 && results.some(r => r.offsetParent !== null); }", "ExtractSearchResults": "() => { const resultsList = document.querySelector('ul[aria-labelledby=\"location-results-title\"]'); if (!resultsList) { console.log('Location results list not found'); return []; } const results = Array.from(resultsList.querySelectorAll('li.bom-linklist__item[role=\"listitem\"]')); console.log('Found', results.length, 'location results'); return results.map((r) => { const nameEl = r.querySelector('[data-testid=\"location-name\"]'); const descEl = r.querySelector('.bom-linklist-item__desc'); const name = nameEl ? (nameEl.textContent || nameEl.innerText || '').trim() : ''; const desc = descEl ? (descEl.textContent || descEl.innerText || '').trim() : ''; const fullText = (r.textContent || r.innerText || '').trim(); console.log('Result:', { hasNameEl: !!nameEl, hasDescEl: !!descEl, name: name, desc: desc }); return [name, desc, fullText]; }); }", "ExtractSearchResultsFallback": "() => { const results = Array.from(document.querySelectorAll('li.bom-linklist__item[role=\"listitem\"]')); return results.map(r => r.textContent || ''); }", "WaitForMapCanvas": "() => { const canvas = document.querySelector('.esri-view-surface canvas'); return canvas && canvas.width > 0 && canvas.height > 0 && canvas.offsetWidth > 0 && canvas.offsetHeight > 0; }", "WaitForEsriView": "() => { try { const elements = document.querySelectorAll('.esri-view'); for (let el of elements) { if (el.__view && el.__view.ready) { return true; } } } catch(e) {} return false; }", "CheckActiveFrameSegment": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); if (segments.length === 0) return true; const activeSegment = segments.find(s => { const style = window.getComputedStyle(s); return style.backgroundColor !== 'rgb(148, 148, 148)' && style.backgroundColor !== 'rgb(148, 148, 148)'; }); return activeSegment && activeSegment.getAttribute('data-id') === '0'; }", "WaitForMapContainer": "() => { const container = document.querySelector('.esri-view-surface'); return container && container.offsetWidth > 0 && container.offsetHeight > 0; }", "ExtractFrameInfo": "() => { const segments = Array.from(document.querySelectorAll('[data-testid=\"bom-scrub-segment\"]')); const now = new Date(); return segments.map((seg, index) => { const ariaLabel = seg.getAttribute('aria-label') || ''; let minutes = null; const timestampMatch = ariaLabel.match(/(?:[A-Za-z]+\\s+)?(\\d{1,2})\\s+([A-Za-z]+),?\\s+(\\d{1,2}):(\\d{2})\\s*(am|pm)/i); if (timestampMatch) { try { const day = parseInt(timestampMatch[1]); const monthStr = timestampMatch[2]; const hour12 = parseInt(timestampMatch[3]); const minute = parseInt(timestampMatch[4]); const ampm = timestampMatch[5].toLowerCase(); const monthNames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']; const month = monthNames.indexOf(monthStr.toLowerCase()); if (month >= 0) { let hour24 = hour12; if (ampm === 'pm' && hour12 !== 12) hour24 += 12; if (ampm === 'am' && hour12 === 12) hour24 = 0; const year = now.getFullYear(); const frameTime = new Date(year, month, day, hour24, minute); if (frameTime > now) frameTime.setFullYear(year - 1); const diffMs = now - frameTime; minutes = Math.round(diffMs / (1000 * 60)); if (minutes < 0 || minutes > 120) minutes = null; } } catch(e) {} } return { index: index, minutesAgo: minutes }; }); }", "CheckModalOverlay": "() => { const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); if (bomOverlay && bomOverlay.style.display !== 'none') { return true; } const recaptchaSelectors = ['.g-recaptcha', '#g-recaptcha', '.rc-anchor-container']; for (const selector of recaptchaSelectors) { const el = document.querySelector(selector); if (el) { const style = window.getComputedStyle(el); const rect = el.getBoundingClientRect(); if (style.display !== 'none' && style.visibility !== 'hidden' && rect.width > 200 && rect.height > 200) { return true; } } } const allForms = document.querySelectorAll('form'); for (const form of allForms) { const action = form.getAttribute('action') || ''; const id = form.getAttribute('id') || ''; if (action.includes('feedback') || id.includes('feedback')) { const style = window.getComputedStyle(form); const rect = form.getBoundingClientRect(); if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { const text = form.textContent || ''; if (text.includes('reCAPTCHA') || text.includes('recaptcha') || text.includes('Tell us why')) { return true; } } } } return false; }", "CheckModalStillVisible": "() => { const bomOverlay = document.querySelector('.bom-modal-overlay--after-open'); if (bomOverlay && bomOverlay.style.display !== 'none') return true; const allForms = document.querySelectorAll('form'); for (const form of allForms) { const action = form.getAttribute('action') || ''; const id = form.getAttribute('id') || ''; if (action.includes('feedback') || id.includes('feedback')) { const style = window.getComputedStyle(form); const rect = form.getBoundingClientRect(); if (style.display !== 'none' && rect.width > 200 && rect.height > 200) { return true; } } } return false; }", "GetViewportSize": "() => JSON.stringify({ width: window.innerWidth, height: window.innerHeight })", "ExtractWeatherMetadata": "() => { const section = document.querySelector('section[data-testid=\"weatherMetadata\"]') || document.querySelector('section[aria-label=\"Last updated\"]'); if (!section) return null; const divs = section.querySelectorAll('div'); return Array.from(divs).map(div => div.textContent.trim()).filter(text => text).join(' '); }" }, "TextPatterns": { "ResultsCountPattern": "(\\d+)\\s+of\\s+(\\d+)", "TimestampPattern": "(?:[A-Za-z]+\\s+)?\\d{1,2}\\s+[A-Za-z]+,?\\s+\\d{1,2}:\\d{2}\\s*(?:am|pm)", "ObservationTimePattern": "Observations:\\s*(\\d+)\\s*minutes?\\s*ago", "ForecastTimePattern": "Forecast:\\s*(\\d+)\\s*minutes?\\s+ago", "ForecastHourAgoPattern": "Forecast:\\s*an\\s+hour\\s+ago", "WeatherStationPattern": "at\\s+([^,]+)\\s+weather\\s+station", "DistancePattern": "(\\d+)\\s*km\\s+from", "ExpectedTexts": { "PlayButtonLabel": "Play", "PauseButtonLabel": "Pause", "RadarLinkText": "Rain radar and weather map" } }, "Workflows": { "RadarScraping": { "Description": "Scrapes radar images for a location", "Steps": { "NavigateHomepage": { "Enabled": true }, "ClickSearchButton": { "Enabled": true }, "FillSearchInput": { "Enabled": true }, "WaitForSearchResults": { "Enabled": true }, "SelectSearchResult": { "Enabled": true }, "ClickRadarLink": { "Enabled": true }, "WaitForMapReady": { "Enabled": true }, "PauseRadar": { "Enabled": true }, "ExtractMetadata": { "Enabled": true }, "ResetToFirstFrame": { "Enabled": true }, "CalculateMapBounds": { "Enabled": true }, "CaptureFrames": { "Enabled": true, "Parameters": { "FrameCount": 7, "WaitBetweenFrames": 5000 } } } }, "TemperatureMap": { "Description": "Scrapes temperature forecast map (future feature)", "Steps": { "NavigateHomepage": { "Enabled": true }, "ClickSearchButton": { "Enabled": true }, "FillSearchInput": { "Enabled": true }, "WaitForSearchResults": { "Enabled": true }, "SelectSearchResult": { "Enabled": true } } } } } }