Implement loading of unauthenticated radar GIS images

2 files changed, 165 insertions, 82 deletions

diff --git a/src/functions/scheduled.ts b/src/functions/scheduled.ts
index ae43469..f92d556 100644
--- a/src/functions/scheduled.ts
+++ b/src/functions/scheduled.ts
@@ -1,51 +1,97 @@
 // Cloudflare Workers scheduled handler for cron jobs
 import type { ScheduledController, ScheduledEvent } from '@cloudflare/workers-types';
+import { DateTime } from 'luxon';
 
 interface Env {
-  BASE_URL: string;
-  AUTH_USER: string;
-  AUTH_PASS: string;
   IMGS_BUCKET: R2Bucket;
 }
 
+interface RadarImage {
+  id: string;
+  forecast_time: string;
+  modified: string;
+  created: string;
+  file_name: string;
+  type: string;
+}
+
+interface RadarResponse {
+  data: {
+    types: {
+      IMSRadar: RadarImage[];
+    };
+  };
+}
+
 
 class GeshemUpdate {
   private bucket: R2Bucket;
-  private env: Env;
+  private static readonly API_URL = 'https://ims.gov.il/he/radar_satellite';
+  private static readonly BASE_URL = 'https://ims.gov.il';
 
-  constructor(bucket: R2Bucket, env: Env) {
+  constructor(bucket: R2Bucket) {
     this.bucket = bucket;
-    this.env = env;
   }
 
-  private async authenticate(request: Request): Promise<Request> {
-    const headers = new Headers(request.headers);
-    const credentials = btoa(`${this.env.AUTH_USER}:${this.env.AUTH_PASS}`);
-    headers.set('Authorization', `Basic ${credentials}`);
+  private convertIsraeliToUTC(dateStr: string, timeStr: string): { date: string; time: string } {
+    // Parse YYYYMMDD HHMM format in Israeli timezone
+    const israeliTime = DateTime.fromFormat(
+      `${dateStr} ${timeStr}`,
+      'yyyyMMdd HHmm',
+      { zone: 'Asia/Jerusalem' }
+    );
 
-    return new Request(request.url, {
-      method: request.method,
-      headers,
-      body: request.body
-    });
+    // Convert to UTC
+    const utcTime = israeliTime.toUTC();
+
+    // Format as YYYYMMDD and HHMM
+    return {
+      date: utcTime.toFormat('yyyyMMdd'),
+      time: utcTime.toFormat('HHmm')
+    };
   }
 
-  private async getLatestImages(): Promise<string[]> {
-    const authenticatedRequest = await this.authenticate(new Request(this.env.BASE_URL));
-    const response = await fetch(authenticatedRequest);
-    const html = await response.text();
+  private async getLatestImage(): Promise<RadarImage | null> {
+    const response = await fetch(GeshemUpdate.API_URL);
 
-    const regex = /radar280comp_\d+\.png/g;
-    const matches = html.match(regex) || [];
-    const uniqueImages = [...new Set(matches)];
+    if (!response.ok) {
+      throw new Error(`Failed to fetch radar data: ${response.statusText}`);
+    }
+
+    const data: RadarResponse = await response.json();
+    const images = data.data.types.IMSRadar;
+
+    if (!images || images.length === 0) {
+      return null;
+    }
 
-    return uniqueImages.sort().slice(-10);
+    // Get the latest image (last in the array)
+    return images[images.length - 1];
   }
 
-  private async getLatestBucketKeys(): Promise<string[]> {
-    const yesterday = new Date();
-    yesterday.setDate(yesterday.getDate() - 1);
-    const yesterdayStr = yesterday.toISOString().slice(0, 10).replace(/-/g, '');
+  private parseFilename(filepath: string): { date: string; time: string } | null {
+    // Extract filename from path: /sites/default/files/ims_data/map_images/IMSRadar4GIS/IMSRadar4GIS_202510030935_0.png
+    const filename = filepath.split('/').pop();
+    if (!filename) return null;
+
+    // Match pattern: IMSRadar4GIS_YYYYMMDDHHMM_0.png
+    const match = filename.match(/IMSRadar4GIS_(\d{8})(\d{4})_\d\.png/);
+    if (!match) return null;
+
+    const dateStr = match[1]; // YYYYMMDD
+    const timeStr = match[2]; // HHMM
+
+    return { date: dateStr, time: timeStr };
+  }
+
+  private async getExistingGisKeys(): Promise<Set<string>> {
+    // Get yesterday's date in YYYYMMDD format
+    const yesterday = DateTime.utc().minus({ days: 1 });
+    const yesterdayStr = yesterday.toFormat('yyyyMMdd');
+
+    console.log(`Listing bucket keys starting from ${yesterdayStr}`);
+
+    const existingKeys = new Set<string>();
 
     try {
       const objects = await this.bucket.list({
@@ -53,42 +99,70 @@ class GeshemUpdate {
         startAfter: `imgs/${yesterdayStr}`
       });
 
-      return objects.objects.map(obj => obj.key);
+      for (const obj of objects.objects) {
+        // Only include keys that end with gis.png
+        if (obj.key.endsWith('gis.png')) {
+          existingKeys.add(obj.key);
+        }
+      }
+
+      console.log(`Found ${existingKeys.size} existing gis.png files since ${yesterdayStr}`);
     } catch (error) {
       console.error('Error listing bucket objects:', error);
-      return [];
     }
+
+    return existingKeys;
   }
 
-  private async fetchMissingImages(): Promise<boolean> {
-    const images = await this.getLatestImages();
-    const s3Images = await this.getLatestBucketKeys();
-    let updated = false;
+  private async fetchAndStoreImages(): Promise<number> {
+    // Get all images from API
+    const apiImage = await this.getLatestImage();
+    if (!apiImage) {
+      console.log('No images found in API response');
+      return 0;
+    }
+
+    // Get existing gis.png keys in bucket
+    const existingKeys = await this.getExistingGisKeys();
 
-    for (const img of images) {
-      const key = this.keyFromFilename(img);
-      if (!s3Images.includes(key)) {
-        await this.fetchImage(img, key);
-        updated = true;
-      }
+    let savedCount = 0;
+
+    console.log(`Processing image: ${apiImage.file_name}`);
+
+    // Parse filename to get date and time
+    const parsed = this.parseFilename(apiImage.file_name);
+    if (!parsed) {
+      console.error(`Failed to parse filename: ${apiImage.file_name}`);
+      return 0;
     }
 
-    return updated;
-  }
+    console.log(`Parsed Israeli time - Date: ${parsed.date}, Time: ${parsed.time}`);
 
-  private async fetchImage(imgName: string, key: string): Promise<void> {
-    console.log(`Downloading ${imgName} from web server`);
+    // Convert Israeli time to UTC
+    const utc = this.convertIsraeliToUTC(parsed.date, parsed.time);
+    console.log(`Converted to UTC - Date: ${utc.date}, Time: ${utc.time}`);
 
-    const imageUrl = `${this.env.BASE_URL}/${imgName}`;
-    const authenticatedRequest = await this.authenticate(new Request(imageUrl));
-    const response = await fetch(authenticatedRequest);
+    // Build R2 key: imgs/YYYYMMDD/HHMM/gis.png
+    const key = `imgs/${utc.date}/${utc.time}/gis.png`;
+
+    // Check if already exists
+    if (existingKeys.has(key)) {
+      console.log(`Image already exists at ${key}`);
+      return 0;
+    }
 
+    // Fetch the image
+    const imageUrl = `${GeshemUpdate.BASE_URL}${apiImage.file_name}`;
+    console.log(`Downloading from ${imageUrl}`);
+
+    const response = await fetch(imageUrl);
     if (!response.ok) {
       throw new Error(`Failed to fetch image: ${response.statusText}`);
     }
 
     const imageData = await response.arrayBuffer();
 
+    // Upload to R2
     console.log(`Uploading to ${key}`);
     await this.bucket.put(key, imageData, {
       httpMetadata: {
@@ -96,30 +170,17 @@ class GeshemUpdate {
         cacheControl: 'public, max-age=31536000'
       }
     });
-  }
-
-  private keyFromFilename(filename: string): string {
-    const [name, dateStr] = filename.split('.')[0].split('_');
-    const res = 280;
 
-    // Parse date: YYYYMMDDHHMM
-    const year = dateStr.slice(0, 4);
-    const month = dateStr.slice(4, 6);
-    const day = dateStr.slice(6, 8);
-    const hour = dateStr.slice(8, 10);
-    const minute = dateStr.slice(10, 12);
+    savedCount++;
 
-    const d = `${year}${month}${day}`;
-    const t = `${hour}${minute}`;
-
-    return `imgs/${d}/${t}/${res}.png`;
+    return savedCount;
   }
 
 
   async run(): Promise<string> {
     try {
-      const updated = await this.fetchMissingImages();
-      return `Updated: ${updated}`;
+      const savedCount = await this.fetchAndStoreImages();
+      return `Saved ${savedCount} new images`;
     } catch (error) {
       console.error('Error in GeshemUpdate.run():', error);
       throw error;
@@ -127,30 +188,34 @@ class GeshemUpdate {
   }
 }
 
+export async function handleSchedule(
+  controller: ScheduledController,
+  env: Env,
+  ctx: ExecutionContext
+): Promise<void> {
+  console.log('Cron job triggered at', new Date().toISOString());
+
+  try {
+    if (!env.IMGS_BUCKET) {
+      throw new Error('R2 bucket not configured');
+    }
+
+    const updater = new GeshemUpdate(env.IMGS_BUCKET);
+    const result = await updater.run();
+
+    console.log(`Cron job completed: ${result}`);
+  } catch (error) {
+    console.error('Error in scheduled function:', error);
+    throw error;
+  }
+}
+
 export default {
   async scheduled(
     controller: ScheduledController,
     env: Env,
     ctx: ExecutionContext
   ): Promise<void> {
-    console.log('Cron job triggered at', new Date().toISOString());
-
-    try {
-      if (!env?.BASE_URL || !env?.AUTH_USER || !env?.AUTH_PASS) {
-        throw new Error('Missing environment variables');
-      }
-
-      if (!env.IMGS_BUCKET) {
-        throw new Error('R2 bucket not configured');
-      }
-
-      const updater = new GeshemUpdate(env.IMGS_BUCKET, env);
-      const result = await updater.run();
-
-      console.log(`Cron job completed: ${result}`);
-    } catch (error) {
-      console.error('Error in scheduled function:', error);
-      throw error;
-    }
+    return handleSchedule(controller, env, ctx);
   }
 };
\ No newline at end of file
diff --git a/src/worker.ts b/src/worker.ts
new file mode 100644
index 0000000..e982634
--- /dev/null
+++ b/src/worker.ts
@@ -0,0 +1,18 @@
+import type { SSRManifest } from 'astro';
+import { App } from 'astro/app';
+import { handle } from '@astrojs/cloudflare/handler';
+import { handleSchedule } from './functions/scheduled';
+
+export function createExports(manifest: SSRManifest) {
+  const app = new App(manifest);
+  return {
+    default: {
+      async fetch(request, env, ctx) {
+        return handle(manifest, app, request, env, ctx);
+      },
+      async scheduled(controller, env, ctx) {
+        return await handleSchedule(controller, env, ctx);
+      }
+    }
+  };
+}