From 7b6314f480d4f7a0932d6463e5341e21566a37c2 Mon Sep 17 00:00:00 2001 From: Yuval Adam <_@yuv.al> Date: Fri, 3 Oct 2025 11:52:01 +0200 Subject: Implement loading of unauthenticated radar GIS images --- src/functions/scheduled.ts | 229 +++++++++++++++++++++++++++++---------------- src/worker.ts | 18 ++++ 2 files changed, 165 insertions(+), 82 deletions(-) create mode 100644 src/worker.ts (limited to 'src') diff --git a/src/functions/scheduled.ts b/src/functions/scheduled.ts index ae43469..f92d556 100644 --- a/src/functions/scheduled.ts +++ b/src/functions/scheduled.ts @@ -1,51 +1,97 @@ // Cloudflare Workers scheduled handler for cron jobs import type { ScheduledController, ScheduledEvent } from '@cloudflare/workers-types'; +import { DateTime } from 'luxon'; interface Env { - BASE_URL: string; - AUTH_USER: string; - AUTH_PASS: string; IMGS_BUCKET: R2Bucket; } +interface RadarImage { + id: string; + forecast_time: string; + modified: string; + created: string; + file_name: string; + type: string; +} + +interface RadarResponse { + data: { + types: { + IMSRadar: RadarImage[]; + }; + }; +} + class GeshemUpdate { private bucket: R2Bucket; - private env: Env; + private static readonly API_URL = 'https://ims.gov.il/he/radar_satellite'; + private static readonly BASE_URL = 'https://ims.gov.il'; - constructor(bucket: R2Bucket, env: Env) { + constructor(bucket: R2Bucket) { this.bucket = bucket; - this.env = env; } - private async authenticate(request: Request): Promise { - const headers = new Headers(request.headers); - const credentials = btoa(`${this.env.AUTH_USER}:${this.env.AUTH_PASS}`); - headers.set('Authorization', `Basic ${credentials}`); + private convertIsraeliToUTC(dateStr: string, timeStr: string): { date: string; time: string } { + // Parse YYYYMMDD HHMM format in Israeli timezone + const israeliTime = DateTime.fromFormat( + `${dateStr} ${timeStr}`, + 'yyyyMMdd HHmm', + { zone: 'Asia/Jerusalem' } + ); + + // Convert to UTC + const utcTime = israeliTime.toUTC(); + + // Format as YYYYMMDD and HHMM + return { + date: utcTime.toFormat('yyyyMMdd'), + time: utcTime.toFormat('HHmm') + }; + } - return new Request(request.url, { - method: request.method, - headers, - body: request.body - }); + private async getLatestImage(): Promise { + const response = await fetch(GeshemUpdate.API_URL); + + if (!response.ok) { + throw new Error(`Failed to fetch radar data: ${response.statusText}`); + } + + const data: RadarResponse = await response.json(); + const images = data.data.types.IMSRadar; + + if (!images || images.length === 0) { + return null; + } + + // Get the latest image (last in the array) + return images[images.length - 1]; } - private async getLatestImages(): Promise { - const authenticatedRequest = await this.authenticate(new Request(this.env.BASE_URL)); - const response = await fetch(authenticatedRequest); - const html = await response.text(); + private parseFilename(filepath: string): { date: string; time: string } | null { + // Extract filename from path: /sites/default/files/ims_data/map_images/IMSRadar4GIS/IMSRadar4GIS_202510030935_0.png + const filename = filepath.split('/').pop(); + if (!filename) return null; + + // Match pattern: IMSRadar4GIS_YYYYMMDDHHMM_0.png + const match = filename.match(/IMSRadar4GIS_(\d{8})(\d{4})_\d\.png/); + if (!match) return null; - const regex = /radar280comp_\d+\.png/g; - const matches = html.match(regex) || []; - const uniqueImages = [...new Set(matches)]; + const dateStr = match[1]; // YYYYMMDD + const timeStr = match[2]; // HHMM - return uniqueImages.sort().slice(-10); + return { date: dateStr, time: timeStr }; } - private async getLatestBucketKeys(): Promise { - const yesterday = new Date(); - yesterday.setDate(yesterday.getDate() - 1); - const yesterdayStr = yesterday.toISOString().slice(0, 10).replace(/-/g, ''); + private async getExistingGisKeys(): Promise> { + // Get yesterday's date in YYYYMMDD format + const yesterday = DateTime.utc().minus({ days: 1 }); + const yesterdayStr = yesterday.toFormat('yyyyMMdd'); + + console.log(`Listing bucket keys starting from ${yesterdayStr}`); + + const existingKeys = new Set(); try { const objects = await this.bucket.list({ @@ -53,42 +99,70 @@ class GeshemUpdate { startAfter: `imgs/${yesterdayStr}` }); - return objects.objects.map(obj => obj.key); + for (const obj of objects.objects) { + // Only include keys that end with gis.png + if (obj.key.endsWith('gis.png')) { + existingKeys.add(obj.key); + } + } + + console.log(`Found ${existingKeys.size} existing gis.png files since ${yesterdayStr}`); } catch (error) { console.error('Error listing bucket objects:', error); - return []; } + + return existingKeys; } - private async fetchMissingImages(): Promise { - const images = await this.getLatestImages(); - const s3Images = await this.getLatestBucketKeys(); - let updated = false; + private async fetchAndStoreImages(): Promise { + // Get all images from API + const apiImage = await this.getLatestImage(); + if (!apiImage) { + console.log('No images found in API response'); + return 0; + } - for (const img of images) { - const key = this.keyFromFilename(img); - if (!s3Images.includes(key)) { - await this.fetchImage(img, key); - updated = true; - } + // Get existing gis.png keys in bucket + const existingKeys = await this.getExistingGisKeys(); + + let savedCount = 0; + + console.log(`Processing image: ${apiImage.file_name}`); + + // Parse filename to get date and time + const parsed = this.parseFilename(apiImage.file_name); + if (!parsed) { + console.error(`Failed to parse filename: ${apiImage.file_name}`); + return 0; } - return updated; - } + console.log(`Parsed Israeli time - Date: ${parsed.date}, Time: ${parsed.time}`); + + // Convert Israeli time to UTC + const utc = this.convertIsraeliToUTC(parsed.date, parsed.time); + console.log(`Converted to UTC - Date: ${utc.date}, Time: ${utc.time}`); + + // Build R2 key: imgs/YYYYMMDD/HHMM/gis.png + const key = `imgs/${utc.date}/${utc.time}/gis.png`; - private async fetchImage(imgName: string, key: string): Promise { - console.log(`Downloading ${imgName} from web server`); + // Check if already exists + if (existingKeys.has(key)) { + console.log(`Image already exists at ${key}`); + return 0; + } - const imageUrl = `${this.env.BASE_URL}/${imgName}`; - const authenticatedRequest = await this.authenticate(new Request(imageUrl)); - const response = await fetch(authenticatedRequest); + // Fetch the image + const imageUrl = `${GeshemUpdate.BASE_URL}${apiImage.file_name}`; + console.log(`Downloading from ${imageUrl}`); + const response = await fetch(imageUrl); if (!response.ok) { throw new Error(`Failed to fetch image: ${response.statusText}`); } const imageData = await response.arrayBuffer(); + // Upload to R2 console.log(`Uploading to ${key}`); await this.bucket.put(key, imageData, { httpMetadata: { @@ -96,30 +170,17 @@ class GeshemUpdate { cacheControl: 'public, max-age=31536000' } }); - } - - private keyFromFilename(filename: string): string { - const [name, dateStr] = filename.split('.')[0].split('_'); - const res = 280; - // Parse date: YYYYMMDDHHMM - const year = dateStr.slice(0, 4); - const month = dateStr.slice(4, 6); - const day = dateStr.slice(6, 8); - const hour = dateStr.slice(8, 10); - const minute = dateStr.slice(10, 12); + savedCount++; - const d = `${year}${month}${day}`; - const t = `${hour}${minute}`; - - return `imgs/${d}/${t}/${res}.png`; + return savedCount; } async run(): Promise { try { - const updated = await this.fetchMissingImages(); - return `Updated: ${updated}`; + const savedCount = await this.fetchAndStoreImages(); + return `Saved ${savedCount} new images`; } catch (error) { console.error('Error in GeshemUpdate.run():', error); throw error; @@ -127,30 +188,34 @@ class GeshemUpdate { } } +export async function handleSchedule( + controller: ScheduledController, + env: Env, + ctx: ExecutionContext +): Promise { + console.log('Cron job triggered at', new Date().toISOString()); + + try { + if (!env.IMGS_BUCKET) { + throw new Error('R2 bucket not configured'); + } + + const updater = new GeshemUpdate(env.IMGS_BUCKET); + const result = await updater.run(); + + console.log(`Cron job completed: ${result}`); + } catch (error) { + console.error('Error in scheduled function:', error); + throw error; + } +} + export default { async scheduled( controller: ScheduledController, env: Env, ctx: ExecutionContext ): Promise { - console.log('Cron job triggered at', new Date().toISOString()); - - try { - if (!env?.BASE_URL || !env?.AUTH_USER || !env?.AUTH_PASS) { - throw new Error('Missing environment variables'); - } - - if (!env.IMGS_BUCKET) { - throw new Error('R2 bucket not configured'); - } - - const updater = new GeshemUpdate(env.IMGS_BUCKET, env); - const result = await updater.run(); - - console.log(`Cron job completed: ${result}`); - } catch (error) { - console.error('Error in scheduled function:', error); - throw error; - } + return handleSchedule(controller, env, ctx); } }; \ No newline at end of file diff --git a/src/worker.ts b/src/worker.ts new file mode 100644 index 0000000..e982634 --- /dev/null +++ b/src/worker.ts @@ -0,0 +1,18 @@ +import type { SSRManifest } from 'astro'; +import { App } from 'astro/app'; +import { handle } from '@astrojs/cloudflare/handler'; +import { handleSchedule } from './functions/scheduled'; + +export function createExports(manifest: SSRManifest) { + const app = new App(manifest); + return { + default: { + async fetch(request, env, ctx) { + return handle(manifest, app, request, env, ctx); + }, + async scheduled(controller, env, ctx) { + return await handleSchedule(controller, env, ctx); + } + } + }; +} -- cgit v1.3.1