Add initial scan script

4 files changed, 59 insertions, 5 deletions

diff --git a/Pipfile b/Pipfile
index 1184c47..230a7f7 100644
--- a/Pipfile
+++ b/Pipfile
@@ -6,9 +6,11 @@ name = "pypi"
 [packages]
 pyyaml = "*"
 jinja2 = "*"
+requests = "*"
 
 [dev-packages]
 
 [scripts]
 build = "python build.py"
-serve = "python -m http.server --directory build"
\ No newline at end of file
+scan = "python scan.py"
+serve = "python -m http.server --directory build"
diff --git a/Pipfile.lock b/Pipfile.lock
index 4df327e..c953db6 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,12 +1,10 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "7345e042a57c26c68acd1a9f08055470fb19f5576161bf55079a250fec526ca1"
+            "sha256": "880f94e4f72127741810d1b63092ea0ce501dafbc84cf3226a08339f76a264df"
         },
         "pipfile-spec": 6,
-        "requires": {
-            "python_version": "3.10"
-        },
+        "requires": {},
         "sources": [
             {
                 "name": "pypi",
@@ -16,6 +14,30 @@
         ]
     },
     "default": {
+        "certifi": {
+            "hashes": [
+                "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14",
+                "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==2022.9.24"
+        },
+        "charset-normalizer": {
+            "hashes": [
+                "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845",
+                "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"
+            ],
+            "markers": "python_full_version >= '3.6.0'",
+            "version": "==2.1.1"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4",
+                "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==3.4"
+        },
         "jinja2": {
             "hashes": [
                 "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852",
@@ -115,6 +137,22 @@
             ],
             "index": "pypi",
             "version": "==6.0"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983",
+                "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"
+            ],
+            "index": "pypi",
+            "version": "==2.28.1"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e",
+                "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'",
+            "version": "==1.26.12"
         }
     },
     "develop": {}
diff --git a/data/.keep b/data/.keep
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/data/.keep
diff --git a/scan.py b/scan.py
new file mode 100644
index 0000000..34c65d8
--- /dev/null
+++ b/scan.py
@@ -0,0 +1,14 @@
+import requests
+
+from pathlib import Path
+
+ROOT_PATH = Path(__file__).parent
+
+DATA_DIR = ROOT_PATH / "data"
+
+TLDS_URL = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
+
+with open(DATA_DIR / "tlds.txt", "w") as f:
+    res = requests.get(TLDS_URL)
+    tlds = [tld.lower() for tld in res.text.split("\n")[1:] if len(tld) < 4]
+    f.write("\n".join(tlds).strip())
\ No newline at end of file