From 0f1ffa5e4534fd777be42f60bdc261ecff879df4 Mon Sep 17 00:00:00 2001
From: Yuval Adam <_@yuv.al>
Date: Thu, 17 Nov 2022 11:44:48 +0200
Subject: Implement fetch_names()

---
 scripts/scan.py | 98 +++++++++++++++++++++++++++++++--------------------------
 1 file changed, 54 insertions(+), 44 deletions(-)

(limited to 'scripts')

diff --git a/scripts/scan.py b/scripts/scan.py
index 62d843a..c4b426a 100644
--- a/scripts/scan.py
+++ b/scripts/scan.py
@@ -1,57 +1,67 @@
+import json
 import requests
 
 from pathlib import Path
+from os import makedirs, listdir
 
 ROOT_PATH = Path(__file__).parents[1]
 
 DATA_DIR = ROOT_PATH / "data"
 
 TLDS_URL = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
-MALE_NAMES_URL = "https://raw.githubusercontent.com/DictionaryHouse/EnglishName/master/top_1000_EN_%E7%94%B7%E6%80%A7names_english.txt"
-FEMALE_NAMES_URL = "https://raw.githubusercontent.com/DictionaryHouse/EnglishName/master/top_1000_EN_%E5%A5%B3%E6%80%A7names_english.txt"
-
-with open(DATA_DIR / "tlds.txt", "w") as f:
-    print("Fetching TLDs...")
-    res = requests.get(TLDS_URL)
-    tlds = [tld.lower() for tld in res.text.strip().split("\n")[1:] if len(tld) < 4]
-    f.write("\n".join(tlds).strip())
-    TLDS = set(tlds)
-
-NAMES = []
-
-with open(DATA_DIR / "names.txt", "w") as f:
-    print("Fetching female names...")
-    res = requests.get(FEMALE_NAMES_URL)
-    f.write(res.text)
-    NAMES += res.text.split("\n")
-    print("Fetching male names...")
-    res = requests.get(MALE_NAMES_URL)
-    f.write(res.text)
-    NAMES += res.text.split("\n")
-
-# print(NAMES)
-# print(TLDS)
-
-CANDIDATES = []
-
-for name in NAMES:
-    if name[-2:] in TLDS:
-        CANDIDATES.append(f"{name[:-2]}.{name[-2:]}")
-    if name[-3:] in TLDS:
-        CANDIDATES.append(f"{name[:-3]}.{name[-3:]}")
-
-with open(DATA_DIR / "domains.txt", "w") as f:
-    f.write("\n".join(CANDIDATES).strip())
-
-for c in CANDIDATES:
-    try:
-        print(f"Fetching {c}...", end="")
-        res = requests.get(f"http://{c}", timeout=5)
+
+NAMES_ENDPOINT = "https://nameberry.com/nameberry/api/v1/search"
+
+
+class NameScanner:
+    def fetch_tlds(self):
+        with open(DATA_DIR / "tlds.txt", "w") as f:
+            print("Fetching TLDs...")
+            res = requests.get(TLDS_URL)
+            tlds = [
+                tld.lower() for tld in res.text.strip().split("\n")[1:] if len(tld) < 4
+            ]
+            f.write("\n".join(tlds).strip())
+            TLDS = set(tlds)
+
+    def fetch_homepage(self, domain):
+        print(f"Fetching {domain}...", end="")
+        res = requests.get(f"http://{domain}", timeout=5)
         if res.ok:
-            print(f"Found {c}!")
-            with open(DATA_DIR / "homepages" / f"{c}.html", "w") as f:
+            print(f"Found {domain}!")
+            with open(DATA_DIR / "homepages" / f"{domain}.html", "w") as f:
                 f.write(res.text)
         else:
             print("x")
-    except:
-        print("x")
\ No newline at end of file
+
+    def fetch_names(self, suffix, count=5000):
+        res = requests.post(
+            NAMES_ENDPOINT,
+            json={
+                "starts_with": "",
+                "ends_with": suffix,
+                "contains": "",
+                "syllables": "",
+                "origin_id": "",
+                "derivation": "",
+                "page": 1,
+                "per_page": count,
+            },
+        )
+        if res.ok:
+            j = res.json()
+            print(f"found {j['advanced_name_count']}")
+            makedirs(DATA_DIR / "names", exist_ok=True)
+            with open(DATA_DIR / "names" / f"{suffix}.json", "w") as f:
+                f.write(res.text)
+
+    def fetch_all(self):
+        with open(DATA_DIR / "tlds.txt", "r") as f:
+            for line in f:
+                d = line.strip()
+                print(f"Fetching {d}...", end="")
+                self.fetch_names(d)
+
+
+ns = NameScanner()
+ns.fetch_names()
-- 
cgit v1.3.1