Make this into more of a CLI thing. Also write some code

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/hilbert.iml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/.venv" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Black">
+    <option name="sdkName" value="Python 3.12 (hilbert)" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.12 (hilbert)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/hilbert.iml" filepath="$PROJECT_DIR$/.idea/hilbert.iml" />
+    </modules>
+  </component>
+</project>

graphs.py

@@ -0,0 +1,156 @@
+import sys
+import numpy as np
+import matplotlib.pyplot as plt
+import matplotlib.colors as colors
+from utils import *
+
+order=16
+grid_size=128
+def line_chart(x,y ):
+    plt.plot(x, y)
+    plt.show()
+
+
+def graph_density(ip_list: str):
+    xarr = []
+    yarr = []
+    for ip in ip_list:
+        try:
+            if ":" in ip:
+                ip = ip_to_int(ip.split(":")[0])
+            else:
+                ip = ip_to_int(ip)
+        except ValueError as e:
+            print(e)
+            continue
+        x, y = d2xy(2**order, ip)
+        xarr.append(x)
+        yarr.append(y)
+
+
+    fig, ax = plt.subplots(figsize=(64, 64))
+
+    # Create density grid
+    H, xedges, yedges = np.histogram2d(xarr, yarr, bins=grid_size, range=[[0, 2**order], [0, 2**order]])
+
+    # Plot heatmap
+    im = ax.imshow(H.T, cmap='YlOrRd', extent=[0, 2**order, 0, 2**order], origin='lower',
+                   norm=colors.LogNorm(vmin=1, vmax=H.max()))
+    # Plot points
+    ax.scatter(xarr, yarr, color='blue', s=1, alpha=0.5)
+
+    ax.set_xlim(0, 2 ** order)
+    ax.set_ylim(0, 2 ** order)
+
+    # plt.colorbar(im, label='Number of IPs')
+    plt.axis('on')
+    plt.show()
+
+def graph_ports(ip_list: str):
+    count = np.zeros(65536)
+
+    for ip in ip_list:
+        if not ":" in ip:
+            continue
+        port = int(ip.split(":")[1])
+        count[port] += 1
+
+    line_chart(np.arange(0, 65536), count)
+
+
+def graph_nonstandard_ports(ip_list: str):
+    xarr = []
+    yarr = []
+    colorarr = []
+    for ip in ip_list:
+        try:
+            if ":" in ip:
+                split = ip.split(":")
+                ip = ip_to_int(split[0])
+                if split[1] == "25565":
+                    colorarr.append('red')
+                else:
+                    colorarr.append('blue')
+
+            else:
+                ip = ip_to_int(ip)
+                colorarr.append('green')
+        except ValueError as e:
+            print(e)
+            continue
+        x, y = d2xy(2**order, ip)
+        xarr.append(x)
+        yarr.append(y)
+
+
+    fig, ax = plt.subplots(figsize=(12, 12))
+
+    # Create density grid
+    H, xedges, yedges = np.histogram2d(xarr, yarr, bins=grid_size, range=[[0, 2**order], [0, 2**order]])
+
+    # Plot heatmap
+    im = ax.imshow(H.T, cmap='YlOrRd', extent=[0, 2**order, 0, 2**order], origin='lower',
+                   norm=colors.LogNorm(vmin=1, vmax=H.max()))
+    # Plot points
+    ax.scatter(xarr, yarr, color=colorarr, s=1, alpha=0.5)
+
+    ax.set_xlim(0, 2 ** order)
+    ax.set_ylim(0, 2 ** order)
+
+    # plt.colorbar(im, label='Number of IPs')
+    plt.axis('on')
+    plt.show()
+
+def random_choord(H):
+    weights = H.flatten()
+
+    weights = weights / np.sum(weights)
+    index = np.random.choice(grid_size*grid_size, p=weights)
+    x, y = divmod(index, grid_size)
+
+    x += np.random.random()
+    y += np.random.random()
+
+    x /= grid_size
+    y /= grid_size
+
+    x *= 2 ** order
+    y *= 2 ** order
+
+    return x, y
+
+
+def generate_random_ips(ip_list: str, count: int):
+    xarr = []
+    yarr = []
+    for ip in ip_list:
+        try:
+            if ":" in ip:
+                split = ip.split(":")
+                ip = ip_to_int(split[0])
+            else:
+                ip = ip_to_int(ip)
+        except ValueError as e:
+            print(e)
+            continue
+        x, y = d2xy(2**order, ip)
+        xarr.append(x)
+        yarr.append(y)
+
+    fig, ax = plt.subplots(figsize=(12, 12))
+
+    H, xedges, yedges = np.histogram2d(xarr, yarr, bins=grid_size, range=[[0, 2**order], [0, 2**order]])
+
+
+    # x /= grid_size
+    # y /= grid_size
+
+    # ips = []
+
+    print(count)
+
+    for i in range(count):
+        x, y = random_choord(H)
+        d = xy2d(2**order, x, y)
+        ip = ip_int_to_ip(d)
+        print(ip)

hilbert_density.py

@@ -1,61 +0,0 @@
-import sys
-import numpy as np
-import matplotlib.pyplot as plt
-import matplotlib.colors as colors
-
-def hilbert_xy(ip_int, order=16):
-    """Convert IP integer to (x,y) on a Hilbert curve of given order."""
-    x = y = 0
-    for i in range(order):
-        xi = (ip_int >> (2 * i)) & 1
-        yi = (ip_int >> (2 * i + 1)) & 1
-
-        if yi == 0:
-            if xi == 1:
-                x = (1 << order) - 1 - x
-                y = (1 << order) - 1 - y
-            x, y = y, x
-
-        x += xi << i
-        y += yi << i
-
-    return x, y
-
-def ip_to_int(ip):
-    return int(''.join([bin(int(x) + 256)[3:] for x in ip.split('.')]), 2)
-
-def create_hilbert_points(ip_list, order=16):
-    points = []
-    for ip in ip_list:
-        ip_int = ip_to_int(ip)
-        x, y = hilbert_xy(ip_int, order)
-        points.append((x, y))
-    return points
-
-def visualize_hilbert_points(ip_list, order=16, grid_size=32):
-    points = create_hilbert_points(ip_list, order)
-    x, y = zip(*points)
-
-    fig, ax = plt.subplots(figsize=(12, 12))
-
-    # Create density grid
-    H, xedges, yedges = np.histogram2d(x, y, bins=grid_size, range=[[0, 2**order], [0, 2**order]])
-
-    # Plot heatmap
-    im = ax.imshow(H.T, cmap='YlOrRd', extent=[0, 2**order, 0, 2**order], origin='lower',
-                   norm=colors.LogNorm(vmin=1, vmax=H.max()))
-
-    # Plot points
-    ax.scatter(x, y, color='blue', s=2, alpha=0.5)
-
-    ax.set_xlim(0, 2 ** order)
-    ax.set_ylim(0, 2 ** order)
-
-    plt.title(f"IP Addresses on Hilbert Curve (order {order}) with Density Heatmap")
-    plt.colorbar(im, label='Number of IPs')
-    plt.axis('off')
-    plt.show()
-
-if __name__ == '__main__':
-    ip_list = open(sys.argv[1]).read().splitlines()
-    visualize_hilbert_points(ip_list)

hilbert_standard_ports.py

@@ -1,37 +0,0 @@
-def inverse_hilbert_xy(x, y, order=16):
-    """Convert (x, y) on a Hilbert curve of given order to IP integer."""
-    ip_int = 0
-    for i in range(order - 1, -1, -1):
-        xi = (x >> i) & 1
-        yi = (y >> i) & 1
-
-        if yi == 0:
-            if xi == 1:
-                x, y = (1 << order) - 1 - y, (1 << order) - 1 - x
-            x, y = y, x
-
-        ip_int |= (xi << (2 * i)) | (yi << (2 * i + 1))
-
-    return ip_int
-
-
-def ip_int_to_ip(ip_int):
-    """Convert an integer IP address to a string IP address."""
-    return ".".join(str((ip_int >> i) & 0xFF) for i in [24, 16, 8, 0])
-
-def generate_ip_ranges(size=16, order=16):
-    # for x in range(size):
-    #     for y in range(size):
-    #         x_pos = int((x/size)*(2 ** order)/2)
-    #         y_pos = int((y/size)*(2 ** order)/2)
-    #         ip = inverse_hilbert_xy(x_pos, y_pos, order=order)
-    #         print(ip_int_to_ip(ip))
-
-    for x in range(256):
-        for y in range(256):
-            print(str(x)+"."+str(y)+".255.255")
-
-
-
-if __name__ == "__main__":
-    generate_ip_ranges()

ip_parser.py

@@ -0,0 +1,63 @@
+import re
+import sys
+
+import utils
+
+
+def process_ips(file_path, verbose=True):
+    ip_port_map = {}
+    ip_regex = r'(?P<ip>(?:\d{1,3}\.){3}\d{1,3})(?::(?P<port>\d{1,5}))?'
+
+    ip_count = 0
+
+    if verbose:
+        print("Reading " + file_path)
+
+    with open(file_path, 'r') as file:
+        for line in file:
+            matches = re.finditer(ip_regex, line)
+            for match in matches:
+                ip = match.group('ip')
+                port = match.group('port')
+
+                ip_count += 1
+
+                if ip in ip_port_map:
+                    if port:
+                        ip_port_map[ip].add(port)
+                else:
+                    ip_port_map[ip] = {port} if port else set()
+
+    unique_ips = []
+    for ip, ports in ip_port_map.items():
+        if ports:  # Only add ports if they exist
+            for port in ports:
+                if port:
+                    unique_ips.append(f"{ip}:{port}")
+                else:
+                    unique_ips.append(ip)
+        else:
+            unique_ips.append(ip)
+    del ip_port_map
+
+    if verbose:
+        print("Removed " + str(ip_count - len(unique_ips)) + " Duplicate IPs")
+
+    # removed_count = 0
+    # valid_ips = []
+    #
+    # for ip in unique_ips:
+    #     if utils.is_valid_ip(ip):
+    #         valid_ips.append(ip)
+    #     else:
+    #         removed_count += 1
+    #         print(ip)
+    #
+    #
+    # del unique_ips
+    # print("Removed " + str(removed_count) + " invalid IPs")
+
+    if verbose:
+        print("Retrieved " + str(len(unique_ips)) + " IPs")
+
+    return unique_ips

main.py

@@ -0,0 +1,43 @@
+import sys
+import traceback
+
+import graphs
+import ip_parser
+order = 16
+
+def printHelp():
+  print("""
+    Usage:
+    
+    ./main.py <command> [options]
+    
+    Commands:
+    parse <input file>
+    graph_density <input file>
+    graph_ports <input file>
+    graph_nonstandard_ports <input file>
+    generate_random_ips <input file> <count>
+  """)
+
+if __name__ == '__main__':
+    try:
+        argv = sys.argv
+        match argv[1]:
+            case 'parse':
+                for ip in ip_parser.process_ips(sys.argv[2], verbose=False):
+                    print(ip)
+            case 'graph_density':
+                graphs.graph_density(ip_parser.process_ips(sys.argv[2]))
+            case 'graph_ports':
+                graphs.graph_ports(ip_parser.process_ips(sys.argv[2]))
+            case 'graph_nonstandard_ports':
+                graphs.graph_nonstandard_ports(ip_parser.process_ips(sys.argv[2]))
+            case 'generate_random_ips':
+                graphs.generate_random_ips(ip_parser.process_ips(sys.argv[2]), int(sys.argv[3]))
+            case _:
+                printHelp()
+    except Exception as e:
+        ex_type, ex, tb = sys.exc_info()
+        traceback.print_tb(tb)
+        print(str(e))
+        printHelp()

temp.py

@@ -1,4 +0,0 @@
-with open("./ips_1.txt") as f:
-    lines = f.readlines()
-    for line in lines:
-        print(line.split("/")[1].split(":")[0].rstrip())

utils.py

@@ -0,0 +1,93 @@
+# https://github.com/bonsaiviking/IPMap/blob/master/ipmap/hilbert.py
+def xy2d(side_length: int, x: int, y: int):
+    """Find the distance of the point (x, y) along a Hilbert curve
+    which fills a square *side_length* units on a side."""
+    y = side_length-y
+    s = side_length / 2
+    d = 0
+    while( s > 0 ):
+        rx = 1 if (int(x) & int(s)) else 0
+        ry = 1 if (int(y) & int(s)) else 0
+        d += s * s * ((3 * rx) ^ ry)
+        if (ry == 0):
+            if (rx == 1):
+                x = s-1 - x
+                y = s-1 - y
+            (x, y) = (y, x)
+        s /= 2
+    return int(d)
+
+# https://github.com/bonsaiviking/IPMap/blob/master/ipmap/hilbert.py
+def d2xy(side_length: int, d: int):
+    """Find the coordinates (x, y) of a point some distance *d* along
+    a Hilbert curve which fills a square *side_length* units on a side."""
+    x = y = 0
+    s = 1
+    while( s < side_length ):
+        rx = 1 & int(d/2)
+        ry = 1 & (int(d) ^ rx)
+        if (ry == 0):
+            if (rx == 1):
+                x = s-1 - x
+                y = s-1 - y
+            (x, y) = (y, x)
+        x += s * rx
+        y += s * ry
+        d /= 4
+        s *= 2
+
+    # if y < 10000:
+    #     print(ip_int_to_ip(int(d)))
+
+    return x, side_length-y
+
+def ip_to_int(ip):
+    return int(''.join([bin(int(x) + 256)[3:] for x in ip.split('.')]), 2)
+
+def ip_int_to_ip(ip_int):
+    return ".".join(str((ip_int >> i) & 0xFF) for i in [24, 16, 8, 0])
+
+
+def ipInCIDR(ip: str, ip_CIDR: str):
+    range_parts = ip_CIDR.split('/')
+    range_mask = int(range_parts[1])
+
+    range_mask_num = (0xFFFFFFFF << (32 - range_mask)) & 0xFFFFFFFF
+
+    return (ip_to_int(ip) & range_mask_num) == (ip_to_int(range_parts[0]) & range_mask_num)
+
+
+def ipInRange(ip: str, ip_range: str):
+    ip_int = ip_to_int(ip)
+    start_ip_int, end_ip_int = [ip_to_int(ip) for ip in ip_range.split('-')]
+
+    return start_ip_int <= ip_int <= end_ip_int
+
+reserved_ip_blocks=[
+    "0.0.0.0/8",
+    "10.0.0.0/8",
+    "100.64.0.0/10",
+    "127.0.0.0/8",
+    "169.254.0.0/16",
+    "172.16.0.0/12",
+    "192.0.0.0/24",
+    "192.0.2.0/24",
+    "192.88.99.0/24",
+    "192.168.0.0/16",
+    "198.18.0.0/15",
+    "198.51.100.0/24",
+    "203.0.113.0/24",
+    "224.0.0.0/4",
+    "233.252.0.0/24",
+    "240.0.0.0/4",
+    "255.255.255.255/32"
+]
+
+def is_valid_ip(ip: str):
+    if ":" in ip:
+        ip = ip.split(":")[0]
+
+    for reserved_ip in reserved_ip_blocks:
+        if ipInCIDR(ip, reserved_ip):
+            return False
+    return True