1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
|
import json
import re
import boto3
import requests
import urllib3
from requests_ntlm import HttpNtlmAuth
from datetime import datetime, timedelta
from io import BytesIO
from os import environ
urllib3.disable_warnings()
class GeshemUpdate():
def __init__(self):
self.s3 = boto3.client('s3')
self.BUCKET = 'imgs.geshem.space'
self.IMG_PREFIX = 'imgs/'
self.BASE_URL = environ.get("BASE_URL")
self.NTLM_AUTH = (environ.get("AUTH_USER"), environ.get("AUTH_PASS"))
self.session = requests.Session()
self.session.auth = HttpNtlmAuth(*self.NTLM_AUTH)
def get_latest_images(self):
imgres = self.session.get(self.BASE_URL, verify=False)
imgs = list(set(re.findall(r"radar280comp_\d+.png", imgres.text)))
return sorted(imgs)[-10:]
def get_latest_bucket_keys(self):
latest_keys = []
yesterday = (datetime.utcnow().date() - timedelta(days=1)).strftime('%Y%m%d')
try:
latest_imgs = self.s3.list_objects_v2(Bucket=self.BUCKET, Prefix=self.IMG_PREFIX, StartAfter=self.IMG_PREFIX + yesterday)['Contents']
latest_keys = [i['Key'] for i in latest_imgs]
except KeyError:
pass
return latest_keys
def fetch_missing_images(self):
imgs = self.get_latest_images()
s3_imgs = self.get_latest_bucket_keys()
updated = False
for img in imgs:
key = self.key_from_filename(img)
if key not in s3_imgs:
self.fetch_image(img, key)
updated = True
return updated
def fetch_image(self, img_name, key):
print(f"Downloading {img_name} from web server")
img = self.session.get(f"{self.BASE_URL}/{img_name}").content
print(f"Uploading to {key}")
self.s3.put_object(Bucket=self.BUCKET, Key=key, Body=img, ContentType='image/png', CacheControl='public, max-age=31536000')
def key_from_filename(self, filename):
_name, date = filename.split(".")[0].split("_")
res = 280
# date conversion isn't really needed but we do it anyway just to make sure
dt = datetime.strptime(date, '%Y%m%d%H%M')
d = dt.strftime('%Y%m%d')
t = dt.strftime('%H%M')
return f'{self.IMG_PREFIX}{d}/{t}/{res}.png'
def generate_json(self):
latest_keys = self.get_latest_bucket_keys()
keys = sorted(list(filter(lambda k: k.endswith('280.png'), latest_keys)))[-10:]
index = json.dumps({'280': keys})
self.s3.put_object(Bucket=self.BUCKET, Key='imgs.json', Body=index, ContentType='application/json', CacheControl='public, max-age=60')
def run(self):
updated = self.fetch_missing_images()
if updated:
self.generate_json()
return f"Updated: {updated}"
def update(event, context):
res = GeshemUpdate().run()
body = {
"message": "SUCCESS: \n" + res,
"input": event
}
response = {
"statusCode": 200,
"body": json.dumps(body)
}
return response
if __name__ == "__main__":
GeshemUpdate().run()
|
