diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/ghpr.py | 26 | ||||
| -rw-r--r-- | scripts/scan.py | 159 |
2 files changed, 137 insertions, 48 deletions
diff --git a/scripts/ghpr.py b/scripts/ghpr.py index 4406ba1..6c9f7f9 100644 --- a/scripts/ghpr.py +++ b/scripts/ghpr.py @@ -11,15 +11,25 @@ with open(path, "r") as f: for line in f: l = line.strip().split(",") domain, handle, name = [x.strip() for x in l] + handle = None if "@" in handle else handle fn = domain.replace(".", "") - with open(ROOT_PATH / "names" / f"{fn}.yml", "w") as out: - s = f"domain: {domain}\nname: {name}\ngithub: {handle}\n" + + path = ROOT_PATH / "names" / f"{fn}.yml" + if path.exists(): + print(f"{domain} already exists") + continue + + with open(path, "w") as out: + s = f"domain: {domain}\nname: {name}\n" + if handle: + s += f"github: {handle}\n" + s += "candidate: true\n" out.write(s) - run(["git", "checkout", "-b", domain]) - run(["git", "add", f"names/{fn}.yml"]) - run(["git", "commit", "-m", f"Add {domain}"]) - run(["git", "push", "-u", "origin", domain]) - run(["gh", "pr", "create", "-t", f"Add {domain}", "-b", f"Hey @{handle}, would you like to merge this PR adding you to https://namehack.club?"]) - run(["git", "checkout", "main"]) + # run(["git", "checkout", "-b", domain]) + # run(["git", "add", f"names/{fn}.yml"]) + # run(["git", "commit", "-m", f"Add {domain}"]) + # run(["git", "push", "-u", "origin", domain]) + # run(["gh", "pr", "create", "-t", f"Add {domain}", "-b", f"Hey @{handle}, would you like to merge this PR adding you to https://namehack.club?"]) + # run(["git", "checkout", "main"]) diff --git a/scripts/scan.py b/scripts/scan.py index 62d843a..24fceb6 100644 --- a/scripts/scan.py +++ b/scripts/scan.py @@ -1,57 +1,136 @@ +import aiohttp +import asyncio +import json import requests from pathlib import Path +from os import makedirs, listdir ROOT_PATH = Path(__file__).parents[1] DATA_DIR = ROOT_PATH / "data" TLDS_URL = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt" -MALE_NAMES_URL = "https://raw.githubusercontent.com/DictionaryHouse/EnglishName/master/top_1000_EN_%E7%94%B7%E6%80%A7names_english.txt" -FEMALE_NAMES_URL = "https://raw.githubusercontent.com/DictionaryHouse/EnglishName/master/top_1000_EN_%E5%A5%B3%E6%80%A7names_english.txt" -with open(DATA_DIR / "tlds.txt", "w") as f: - print("Fetching TLDs...") - res = requests.get(TLDS_URL) - tlds = [tld.lower() for tld in res.text.strip().split("\n")[1:] if len(tld) < 4] - f.write("\n".join(tlds).strip()) - TLDS = set(tlds) +NAMES_ENDPOINT = "https://nameberry.com/nameberry/api/v1/search" -NAMES = [] -with open(DATA_DIR / "names.txt", "w") as f: - print("Fetching female names...") - res = requests.get(FEMALE_NAMES_URL) - f.write(res.text) - NAMES += res.text.split("\n") - print("Fetching male names...") - res = requests.get(MALE_NAMES_URL) - f.write(res.text) - NAMES += res.text.split("\n") +class NameScanner: + def fetch_tlds(self): + with open(DATA_DIR / "tlds.txt", "w") as f: + print("Fetching TLDs...") + res = requests.get(TLDS_URL) + tlds = [ + tld.lower() for tld in res.text.strip().split("\n")[1:] if len(tld) < 4 + ] + f.write("\n".join(tlds).strip()) + TLDS = set(tlds) -# print(NAMES) -# print(TLDS) - -CANDIDATES = [] - -for name in NAMES: - if name[-2:] in TLDS: - CANDIDATES.append(f"{name[:-2]}.{name[-2:]}") - if name[-3:] in TLDS: - CANDIDATES.append(f"{name[:-3]}.{name[-3:]}") - -with open(DATA_DIR / "domains.txt", "w") as f: - f.write("\n".join(CANDIDATES).strip()) - -for c in CANDIDATES: - try: - print(f"Fetching {c}...", end="") - res = requests.get(f"http://{c}", timeout=5) + def fetch_homepage(self, domain): + print(f"Fetching {domain}...", end="") + res = requests.get(f"http://{domain}", timeout=5) if res.ok: - print(f"Found {c}!") - with open(DATA_DIR / "homepages" / f"{c}.html", "w") as f: + print(f"Found {domain}!") + with open(DATA_DIR / "homepages" / f"{domain}.html", "w") as f: f.write(res.text) else: print("x") - except: - print("x")
\ No newline at end of file + + def fetch_names(self, suffix, count=5000): + res = requests.post( + NAMES_ENDPOINT, + json={ + "starts_with": "", + "ends_with": suffix, + "contains": "", + "syllables": "", + "origin_id": "", + "derivation": "", + "page": 1, + "per_page": count, + }, + ) + if res.ok: + j = res.json() + print(f"found {j['advanced_name_count']}") + makedirs(DATA_DIR / "names", exist_ok=True) + with open(DATA_DIR / "names" / f"{suffix}.json", "w") as f: + f.write(res.text) + + def fetch_all(self): + with open(DATA_DIR / "tlds.txt", "r") as f: + for line in f: + d = line.strip() + print(f"Fetching {d}...", end="") + self.fetch_names(d) + + def consolidate_names(self): + res = set() + for fn in listdir(DATA_DIR / "names"): + with open(DATA_DIR / "names" / fn, "r") as f: + j = json.load(f) + names = j["advanced"] + if names: + suffix = fn.split(".")[0] + for name in names: + n = name["name"].lower().split(suffix)[0] + if n: + res.add(n) + with open(DATA_DIR / "allnames.txt", "w") as out: + for n in res: + out.write(n + f".{suffix}\n") + + async def get_homepage(self, session, domain): + try: + url = f"http://{domain}" + async with session.get(url) as res: + res = await res.text() + if res: + print(f"Got {len(res)} bytes from {domain}") + with open(DATA_DIR / "homepages" / f"{domain}.html", "w") as out: + out.write(res) + return len(res) + except Exception as e: + return 0 + + async def fetch_homepages(self, N): + with open(DATA_DIR / "allnames.txt", "r") as f: + names = [x.strip() for x in f.readlines()][N * 1000 : 1000 * (N + 1)] + + timeout = aiohttp.ClientTimeout(total=30) + async with aiohttp.ClientSession(timeout=timeout) as session: + tasks = [] + for name in names: + tasks.append(asyncio.ensure_future(self.get_homepage(session, name))) + + resps = await asyncio.gather(*tasks) + + def find_homepages(self): + res = {} + for fn in listdir(DATA_DIR / "homepages"): + with open(DATA_DIR / "homepages" / fn, "r") as f: + domain = fn[:-5] + name = domain.replace(".", "") + text = f.read() + nip = name in text + github = "github" in text + li = "linkedin" in text + sale = "for sale" in text or "register" in text or "parking" in text + res["domain"] = { + "name": name, + "name_in_page": nip, + "github": github, + "linkedin": li, + } + ge = "✅" if github else " " + le = "✅" if li else " " + ne = "✅" if nip else " " + se = "❌" if sale else " " + print(f"{ne}{ge}{le}{se}{name:20}") + + +ns = NameScanner() +# for i in range(0, 26): +# asyncio.run(ns.fetch_homepages(i)) +# ns.consolidate_names() +ns.find_homepages() |