Ignore incapsula failed requests

1 files changed, 13 insertions, 5 deletions

diff --git a/pakarlib/fetch.py b/pakarlib/fetch.py
index 3f9336b..31b9c62 100644
--- a/pakarlib/fetch.py
+++ b/pakarlib/fetch.py
@@ -4,6 +4,7 @@ from pathlib import Path
 
 DATA_DIR = Path(__file__).parent / "data"
 
+
 def get_cities():
     print("Fetching cities...")
     LANGS = ["he", "ar", "en", "ru"]
@@ -14,9 +15,10 @@ def get_cities():
         if "Access Denied" in res.text:
             print("Access denied, must run from Israeli IP")
             break
-        with open(DATA_DIR / "cities"/ f"{lang}.json", "wb") as f:
+        with open(DATA_DIR / "cities" / f"{lang}.json", "wb") as f:
             f.write(res.content)
 
+
 def get_districts():
     print("Fetching districts...")
     LANGS = ["he", "ar", "en", "ru"]
@@ -27,9 +29,10 @@ def get_districts():
         if "Access Denied" in res.text:
             print("Access denied, must run from Israeli IP")
             break
-        with open(DATA_DIR / "districts"/ f"{lang}.json", "wb") as f:
+        with open(DATA_DIR / "districts" / f"{lang}.json", "wb") as f:
             f.write(res.content)
 
+
 def get_segments():
     print("Fetching segments...")
     LOCALES = ["iw_IL", "en_US", "ru_RU", "ar_EG"]
@@ -41,22 +44,27 @@ def get_segments():
         with open(DATA_DIR / "segments" / f"{locale[:2]}.json", "wb") as f:
             f.write(res.content)
 
+
 def get_polygons():
     print("Fetching segments...")
     BASE_URL = "https://dist.meser-hadash.org.il/smart-dist/services/anonymous/polygon/id/android"
     sess = requests.Session()
-    
+
     for i in range(5000000, 5005000):
-        res = sess.get(BASE_URL, params={"id": str(i), "instance": "1544803905", "locale": "iw_IL"})
+        res = sess.get(
+            BASE_URL, params={"id": str(i), "instance": "1544803905", "locale": "iw_IL"}
+        )
         if "CDATA" in res.text:
-            print(res.text)
             print(f"{i} - got CDATA error, skipping")
+        if "Request unsuccessful" in res.text:
+            print(f"{i} - got incapsula error, skipping")
         else:
             with open(DATA_DIR / "polygons" / f"{i}.json", "wb") as f:
                 cont = res.content
                 print(f"{i} - writing {len(cont)} bytes")
                 f.write(cont)
 
+
 if __name__ == "__main__":
     get_cities()
     get_districts()