From 6de7cb6ec533c9917441cbd4b9a1807f8f1cbe16 Mon Sep 17 00:00:00 2001 From: Yuval Adam <_@yuv.al> Date: Tue, 2 May 2023 10:23:30 +0300 Subject: Migrate scan script --- cli/__init__.py | 4 +- cli/ghpr.py | 35 ++++++++++++++ cli/scan.py | 140 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ scripts/ghpr.py | 35 -------------- scripts/scan.py | 136 ------------------------------------------------------ 5 files changed, 178 insertions(+), 172 deletions(-) create mode 100644 cli/ghpr.py create mode 100644 cli/scan.py delete mode 100644 scripts/ghpr.py delete mode 100644 scripts/scan.py diff --git a/cli/__init__.py b/cli/__init__.py index 33e2b10..0b67a9e 100644 --- a/cli/__init__.py +++ b/cli/__init__.py @@ -1,3 +1,5 @@ from .base import cli -from .clean import clean + from .build import build +from .clean import clean +from .scan import scan diff --git a/cli/ghpr.py b/cli/ghpr.py new file mode 100644 index 0000000..6c9f7f9 --- /dev/null +++ b/cli/ghpr.py @@ -0,0 +1,35 @@ +import sys + +from pathlib import Path +from subprocess import run + +ROOT_PATH = Path(__file__).parents[1] + +path = sys.argv[1] + +with open(path, "r") as f: + for line in f: + l = line.strip().split(",") + domain, handle, name = [x.strip() for x in l] + handle = None if "@" in handle else handle + + fn = domain.replace(".", "") + + path = ROOT_PATH / "names" / f"{fn}.yml" + if path.exists(): + print(f"{domain} already exists") + continue + + with open(path, "w") as out: + s = f"domain: {domain}\nname: {name}\n" + if handle: + s += f"github: {handle}\n" + s += "candidate: true\n" + out.write(s) + + # run(["git", "checkout", "-b", domain]) + # run(["git", "add", f"names/{fn}.yml"]) + # run(["git", "commit", "-m", f"Add {domain}"]) + # run(["git", "push", "-u", "origin", domain]) + # run(["gh", "pr", "create", "-t", f"Add {domain}", "-b", f"Hey @{handle}, would you like to merge this PR adding you to https://namehack.club?"]) + # run(["git", "checkout", "main"]) diff --git a/cli/scan.py b/cli/scan.py new file mode 100644 index 0000000..6319d81 --- /dev/null +++ b/cli/scan.py @@ -0,0 +1,140 @@ +import aiohttp +import asyncio +import json +import requests + +from pathlib import Path +from os import makedirs, listdir + +from .base import cli + +ROOT_PATH = Path(__file__).parents[1] + +DATA_DIR = ROOT_PATH / "data" + +TLDS_URL = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt" + +NAMES_ENDPOINT = "https://nameberry.com/nameberry/api/v1/search" + + +class NameScanner: + def fetch_tlds(self): + with open(DATA_DIR / "tlds.txt", "w") as f: + print("Fetching TLDs...") + res = requests.get(TLDS_URL) + tlds = [ + tld.lower() for tld in res.text.strip().split("\n")[1:] if len(tld) < 4 + ] + f.write("\n".join(tlds).strip()) + TLDS = set(tlds) + + def fetch_homepage(self, domain): + print(f"Fetching {domain}...", end="") + res = requests.get(f"http://{domain}", timeout=5) + if res.ok: + print(f"Found {domain}!") + with open(DATA_DIR / "homepages" / f"{domain}.html", "w") as f: + f.write(res.text) + else: + print("x") + + def fetch_names(self, suffix, count=5000): + res = requests.post( + NAMES_ENDPOINT, + json={ + "starts_with": "", + "ends_with": suffix, + "contains": "", + "syllables": "", + "origin_id": "", + "derivation": "", + "page": 1, + "per_page": count, + }, + ) + if res.ok: + j = res.json() + print(f"found {j['advanced_name_count']}") + makedirs(DATA_DIR / "names", exist_ok=True) + with open(DATA_DIR / "names" / f"{suffix}.json", "w") as f: + f.write(res.text) + + def fetch_all(self): + with open(DATA_DIR / "tlds.txt", "r") as f: + for line in f: + d = line.strip() + print(f"Fetching {d}...", end="") + self.fetch_names(d) + + def consolidate_names(self): + res = set() + for fn in listdir(DATA_DIR / "names"): + with open(DATA_DIR / "names" / fn, "r") as f: + j = json.load(f) + names = j["advanced"] + if names: + suffix = fn.split(".")[0] + for name in names: + n = name["name"].lower().split(suffix)[0] + if n: + res.add(n) + with open(DATA_DIR / "allnames.txt", "w") as out: + for n in res: + out.write(n + f".{suffix}\n") + + async def get_homepage(self, session, domain): + try: + url = f"http://{domain}" + async with session.get(url) as res: + res = await res.text() + if res: + print(f"Got {len(res)} bytes from {domain}") + with open(DATA_DIR / "homepages" / f"{domain}.html", "w") as out: + out.write(res) + return len(res) + except Exception as e: + return 0 + + async def fetch_homepages(self, N): + with open(DATA_DIR / "allnames.txt", "r") as f: + names = [x.strip() for x in f.readlines()][N * 1000 : 1000 * (N + 1)] + + timeout = aiohttp.ClientTimeout(total=30) + async with aiohttp.ClientSession(timeout=timeout) as session: + tasks = [] + for name in names: + tasks.append(asyncio.ensure_future(self.get_homepage(session, name))) + + resps = await asyncio.gather(*tasks) + + def find_homepages(self): + res = {} + for fn in listdir(DATA_DIR / "homepages"): + with open(DATA_DIR / "homepages" / fn, "r") as f: + domain = fn[:-5] + name = domain.replace(".", "") + text = f.read() + nip = name in text + github = "github" in text + li = "linkedin" in text + sale = "for sale" in text or "register" in text or "parking" in text + res["domain"] = { + "name": name, + "name_in_page": nip, + "github": github, + "linkedin": li, + } + ge = "✅" if github else " " + le = "✅" if li else " " + ne = "✅" if nip else " " + se = "❌" if sale else " " + print(f"{ne}{ge}{le}{se}{name:20}") + + +@cli.command() +def scan(): + ns = NameScanner() + # for i in range(0, 26): + # asyncio.run(ns.fetch_homepages(i)) + # ns.consolidate_names() + ns.find_homepages() diff --git a/scripts/ghpr.py b/scripts/ghpr.py deleted file mode 100644 index 6c9f7f9..0000000 --- a/scripts/ghpr.py +++ /dev/null @@ -1,35 +0,0 @@ -import sys - -from pathlib import Path -from subprocess import run - -ROOT_PATH = Path(__file__).parents[1] - -path = sys.argv[1] - -with open(path, "r") as f: - for line in f: - l = line.strip().split(",") - domain, handle, name = [x.strip() for x in l] - handle = None if "@" in handle else handle - - fn = domain.replace(".", "") - - path = ROOT_PATH / "names" / f"{fn}.yml" - if path.exists(): - print(f"{domain} already exists") - continue - - with open(path, "w") as out: - s = f"domain: {domain}\nname: {name}\n" - if handle: - s += f"github: {handle}\n" - s += "candidate: true\n" - out.write(s) - - # run(["git", "checkout", "-b", domain]) - # run(["git", "add", f"names/{fn}.yml"]) - # run(["git", "commit", "-m", f"Add {domain}"]) - # run(["git", "push", "-u", "origin", domain]) - # run(["gh", "pr", "create", "-t", f"Add {domain}", "-b", f"Hey @{handle}, would you like to merge this PR adding you to https://namehack.club?"]) - # run(["git", "checkout", "main"]) diff --git a/scripts/scan.py b/scripts/scan.py deleted file mode 100644 index 24fceb6..0000000 --- a/scripts/scan.py +++ /dev/null @@ -1,136 +0,0 @@ -import aiohttp -import asyncio -import json -import requests - -from pathlib import Path -from os import makedirs, listdir - -ROOT_PATH = Path(__file__).parents[1] - -DATA_DIR = ROOT_PATH / "data" - -TLDS_URL = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt" - -NAMES_ENDPOINT = "https://nameberry.com/nameberry/api/v1/search" - - -class NameScanner: - def fetch_tlds(self): - with open(DATA_DIR / "tlds.txt", "w") as f: - print("Fetching TLDs...") - res = requests.get(TLDS_URL) - tlds = [ - tld.lower() for tld in res.text.strip().split("\n")[1:] if len(tld) < 4 - ] - f.write("\n".join(tlds).strip()) - TLDS = set(tlds) - - def fetch_homepage(self, domain): - print(f"Fetching {domain}...", end="") - res = requests.get(f"http://{domain}", timeout=5) - if res.ok: - print(f"Found {domain}!") - with open(DATA_DIR / "homepages" / f"{domain}.html", "w") as f: - f.write(res.text) - else: - print("x") - - def fetch_names(self, suffix, count=5000): - res = requests.post( - NAMES_ENDPOINT, - json={ - "starts_with": "", - "ends_with": suffix, - "contains": "", - "syllables": "", - "origin_id": "", - "derivation": "", - "page": 1, - "per_page": count, - }, - ) - if res.ok: - j = res.json() - print(f"found {j['advanced_name_count']}") - makedirs(DATA_DIR / "names", exist_ok=True) - with open(DATA_DIR / "names" / f"{suffix}.json", "w") as f: - f.write(res.text) - - def fetch_all(self): - with open(DATA_DIR / "tlds.txt", "r") as f: - for line in f: - d = line.strip() - print(f"Fetching {d}...", end="") - self.fetch_names(d) - - def consolidate_names(self): - res = set() - for fn in listdir(DATA_DIR / "names"): - with open(DATA_DIR / "names" / fn, "r") as f: - j = json.load(f) - names = j["advanced"] - if names: - suffix = fn.split(".")[0] - for name in names: - n = name["name"].lower().split(suffix)[0] - if n: - res.add(n) - with open(DATA_DIR / "allnames.txt", "w") as out: - for n in res: - out.write(n + f".{suffix}\n") - - async def get_homepage(self, session, domain): - try: - url = f"http://{domain}" - async with session.get(url) as res: - res = await res.text() - if res: - print(f"Got {len(res)} bytes from {domain}") - with open(DATA_DIR / "homepages" / f"{domain}.html", "w") as out: - out.write(res) - return len(res) - except Exception as e: - return 0 - - async def fetch_homepages(self, N): - with open(DATA_DIR / "allnames.txt", "r") as f: - names = [x.strip() for x in f.readlines()][N * 1000 : 1000 * (N + 1)] - - timeout = aiohttp.ClientTimeout(total=30) - async with aiohttp.ClientSession(timeout=timeout) as session: - tasks = [] - for name in names: - tasks.append(asyncio.ensure_future(self.get_homepage(session, name))) - - resps = await asyncio.gather(*tasks) - - def find_homepages(self): - res = {} - for fn in listdir(DATA_DIR / "homepages"): - with open(DATA_DIR / "homepages" / fn, "r") as f: - domain = fn[:-5] - name = domain.replace(".", "") - text = f.read() - nip = name in text - github = "github" in text - li = "linkedin" in text - sale = "for sale" in text or "register" in text or "parking" in text - res["domain"] = { - "name": name, - "name_in_page": nip, - "github": github, - "linkedin": li, - } - ge = "✅" if github else " " - le = "✅" if li else " " - ne = "✅" if nip else " " - se = "❌" if sale else " " - print(f"{ne}{ge}{le}{se}{name:20}") - - -ns = NameScanner() -# for i in range(0, 26): -# asyncio.run(ns.fetch_homepages(i)) -# ns.consolidate_names() -ns.find_homepages() -- cgit v1.3.1