blob: cf367f490a9653463a848d82b34fb7757cf55aa9 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
|
import requests
from pathlib import Path
ROOT_PATH = Path(__file__).parent
DATA_DIR = ROOT_PATH / "data"
TLDS_URL = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
MALE_NAMES_URL = "https://raw.githubusercontent.com/DictionaryHouse/EnglishName/master/top_1000_EN_%E7%94%B7%E6%80%A7names_english.txt"
FEMALE_NAMES_URL = "https://raw.githubusercontent.com/DictionaryHouse/EnglishName/master/top_1000_EN_%E5%A5%B3%E6%80%A7names_english.txt"
with open(DATA_DIR / "tlds.txt", "w") as f:
print("Fetching TLDs...")
res = requests.get(TLDS_URL)
tlds = [tld.lower() for tld in res.text.strip().split("\n")[1:] if len(tld) < 4]
f.write("\n".join(tlds).strip())
TLDS = set(tlds)
NAMES = []
with open(DATA_DIR / "names.txt", "w") as f:
print("Fetching female names...")
res = requests.get(FEMALE_NAMES_URL)
f.write(res.text)
NAMES += res.text.split("\n")
print("Fetching male names...")
res = requests.get(MALE_NAMES_URL)
f.write(res.text)
NAMES += res.text.split("\n")
# print(NAMES)
# print(TLDS)
CANDIDATES = []
for name in NAMES:
if name[-2:] in TLDS:
CANDIDATES.append(f"{name[:-2]}.{name[-2:]}")
if name[-3:] in TLDS:
CANDIDATES.append(f"{name[:-3]}.{name[-3:]}")
with open(DATA_DIR / "domains.txt", "w") as f:
f.write("\n".join(CANDIDATES).strip())
for c in CANDIDATES:
try:
print(f"Fetching {c}...", end="")
res = requests.get(f"http://{c}", timeout=5)
if res.ok:
print(f"Found {c}!")
with open(DATA_DIR / "homepages" / f"{c}.html", "w") as f:
f.write(res.text)
else:
print("x")
except:
print("x")
|
