diff options
Diffstat (limited to 'scripts/scan.py')
| -rw-r--r-- | scripts/scan.py | 90 |
1 files changed, 50 insertions, 40 deletions
diff --git a/scripts/scan.py b/scripts/scan.py index 62d843a..c4b426a 100644 --- a/scripts/scan.py +++ b/scripts/scan.py @@ -1,57 +1,67 @@ +import json import requests from pathlib import Path +from os import makedirs, listdir ROOT_PATH = Path(__file__).parents[1] DATA_DIR = ROOT_PATH / "data" TLDS_URL = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt" -MALE_NAMES_URL = "https://raw.githubusercontent.com/DictionaryHouse/EnglishName/master/top_1000_EN_%E7%94%B7%E6%80%A7names_english.txt" -FEMALE_NAMES_URL = "https://raw.githubusercontent.com/DictionaryHouse/EnglishName/master/top_1000_EN_%E5%A5%B3%E6%80%A7names_english.txt" -with open(DATA_DIR / "tlds.txt", "w") as f: - print("Fetching TLDs...") - res = requests.get(TLDS_URL) - tlds = [tld.lower() for tld in res.text.strip().split("\n")[1:] if len(tld) < 4] - f.write("\n".join(tlds).strip()) - TLDS = set(tlds) +NAMES_ENDPOINT = "https://nameberry.com/nameberry/api/v1/search" -NAMES = [] -with open(DATA_DIR / "names.txt", "w") as f: - print("Fetching female names...") - res = requests.get(FEMALE_NAMES_URL) - f.write(res.text) - NAMES += res.text.split("\n") - print("Fetching male names...") - res = requests.get(MALE_NAMES_URL) - f.write(res.text) - NAMES += res.text.split("\n") +class NameScanner: + def fetch_tlds(self): + with open(DATA_DIR / "tlds.txt", "w") as f: + print("Fetching TLDs...") + res = requests.get(TLDS_URL) + tlds = [ + tld.lower() for tld in res.text.strip().split("\n")[1:] if len(tld) < 4 + ] + f.write("\n".join(tlds).strip()) + TLDS = set(tlds) -# print(NAMES) -# print(TLDS) - -CANDIDATES = [] - -for name in NAMES: - if name[-2:] in TLDS: - CANDIDATES.append(f"{name[:-2]}.{name[-2:]}") - if name[-3:] in TLDS: - CANDIDATES.append(f"{name[:-3]}.{name[-3:]}") - -with open(DATA_DIR / "domains.txt", "w") as f: - f.write("\n".join(CANDIDATES).strip()) - -for c in CANDIDATES: - try: - print(f"Fetching {c}...", end="") - res = requests.get(f"http://{c}", timeout=5) + def fetch_homepage(self, domain): + print(f"Fetching {domain}...", end="") + res = requests.get(f"http://{domain}", timeout=5) if res.ok: - print(f"Found {c}!") - with open(DATA_DIR / "homepages" / f"{c}.html", "w") as f: + print(f"Found {domain}!") + with open(DATA_DIR / "homepages" / f"{domain}.html", "w") as f: f.write(res.text) else: print("x") - except: - print("x")
\ No newline at end of file + + def fetch_names(self, suffix, count=5000): + res = requests.post( + NAMES_ENDPOINT, + json={ + "starts_with": "", + "ends_with": suffix, + "contains": "", + "syllables": "", + "origin_id": "", + "derivation": "", + "page": 1, + "per_page": count, + }, + ) + if res.ok: + j = res.json() + print(f"found {j['advanced_name_count']}") + makedirs(DATA_DIR / "names", exist_ok=True) + with open(DATA_DIR / "names" / f"{suffix}.json", "w") as f: + f.write(res.text) + + def fetch_all(self): + with open(DATA_DIR / "tlds.txt", "r") as f: + for line in f: + d = line.strip() + print(f"Fetching {d}...", end="") + self.fetch_names(d) + + +ns = NameScanner() +ns.fetch_names() |