astatin3/ip-hilbert
1
0
Fork 0
mirror of https://github.com/Astatin3/ip-hilbert.git synced 2026-06-08 16:08:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

Make this into more of a CLI thing. Also write some code

Browse Source
This commit is contained in:
astatin3
2024-07-10 20:39:15 -06:00
parent 3377dd7e9a
commit 3662b18e9b
14 changed files with 394 additions and 73692 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.idea/.gitignore
Generated
+8
View File
@@ -0,0 +1,8 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml
.idea/hilbert.iml
Generated
+10
View File
@@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/.venv" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
.idea/inspectionProfiles/profiles_settings.xml
Generated
+6
View File
@@ -0,0 +1,6 @@
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>
.idea/misc.xml
Generated
+7
View File
@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Black">
<option name="sdkName" value="Python 3.12 (hilbert)" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.12 (hilbert)" project-jdk-type="Python SDK" />
</project>
.idea/modules.xml
Generated
+8
View File
@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/hilbert.iml" filepath="$PROJECT_DIR$/.idea/hilbert.iml" />
</modules>
</component>
</project>
graphs.py
+156
View File
@@ -0,0 +1,156 @@
import sys
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.colors as colors
from utils import *
order=16
grid_size=128
def line_chart(x,y ):
plt.plot(x, y)
plt.show()
def graph_density(ip_list: str):
xarr = []
yarr = []
for ip in ip_list:
try:
if ":" in ip:
ip = ip_to_int(ip.split(":")[0])
else:
ip = ip_to_int(ip)
except ValueError as e:
print(e)
continue
x, y = d2xy(2**order, ip)
xarr.append(x)
yarr.append(y)
fig, ax = plt.subplots(figsize=(64, 64))
# Create density grid
H, xedges, yedges = np.histogram2d(xarr, yarr, bins=grid_size, range=[[0, 2**order], [0, 2**order]])
# Plot heatmap
im = ax.imshow(H.T, cmap='YlOrRd', extent=[0, 2**order, 0, 2**order], origin='lower',
norm=colors.LogNorm(vmin=1, vmax=H.max()))
# Plot points
ax.scatter(xarr, yarr, color='blue', s=1, alpha=0.5)
ax.set_xlim(0, 2 ** order)
ax.set_ylim(0, 2 ** order)
# plt.colorbar(im, label='Number of IPs')
plt.axis('on')
plt.show()
def graph_ports(ip_list: str):
count = np.zeros(65536)
for ip in ip_list:
if not ":" in ip:
continue
port = int(ip.split(":")[1])
count[port] += 1
line_chart(np.arange(0, 65536), count)
def graph_nonstandard_ports(ip_list: str):
xarr = []
yarr = []
colorarr = []
for ip in ip_list:
try:
if ":" in ip:
split = ip.split(":")
ip = ip_to_int(split[0])
if split[1] == "25565":
colorarr.append('red')
else:
colorarr.append('blue')
else:
ip = ip_to_int(ip)
colorarr.append('green')
except ValueError as e:
print(e)
continue
x, y = d2xy(2**order, ip)
xarr.append(x)
yarr.append(y)
fig, ax = plt.subplots(figsize=(12, 12))
# Create density grid
H, xedges, yedges = np.histogram2d(xarr, yarr, bins=grid_size, range=[[0, 2**order], [0, 2**order]])
# Plot heatmap
im = ax.imshow(H.T, cmap='YlOrRd', extent=[0, 2**order, 0, 2**order], origin='lower',
norm=colors.LogNorm(vmin=1, vmax=H.max()))
# Plot points
ax.scatter(xarr, yarr, color=colorarr, s=1, alpha=0.5)
ax.set_xlim(0, 2 ** order)
ax.set_ylim(0, 2 ** order)
# plt.colorbar(im, label='Number of IPs')
plt.axis('on')
plt.show()
def random_choord(H):
weights = H.flatten()
weights = weights / np.sum(weights)
index = np.random.choice(grid_size*grid_size, p=weights)
x, y = divmod(index, grid_size)
x += np.random.random()
y += np.random.random()
x /= grid_size
y /= grid_size
x *= 2 ** order
y *= 2 ** order
return x, y
def generate_random_ips(ip_list: str, count: int):
xarr = []
yarr = []
for ip in ip_list:
try:
if ":" in ip:
split = ip.split(":")
ip = ip_to_int(split[0])
else:
ip = ip_to_int(ip)
except ValueError as e:
print(e)
continue
x, y = d2xy(2**order, ip)
xarr.append(x)
yarr.append(y)
fig, ax = plt.subplots(figsize=(12, 12))
H, xedges, yedges = np.histogram2d(xarr, yarr, bins=grid_size, range=[[0, 2**order], [0, 2**order]])
# x /= grid_size
# y /= grid_size
# ips = []
print(count)
for i in range(count):
x, y = random_choord(H)
d = xy2d(2**order, x, y)
ip = ip_int_to_ip(d)
print(ip)
hilbert_density.py
-61
View File
@@ -1,61 +0,0 @@
import sys
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.colors as colors
def hilbert_xy(ip_int, order=16):
"""Convert IP integer to (x,y) on a Hilbert curve of given order."""
x = y = 0
for i in range(order):
xi = (ip_int >> (2 * i)) & 1
yi = (ip_int >> (2 * i + 1)) & 1
if yi == 0:
if xi == 1:
x = (1 << order) - 1 - x
y = (1 << order) - 1 - y
x, y = y, x
x += xi << i
y += yi << i
return x, y
def ip_to_int(ip):
return int(''.join([bin(int(x) + 256)[3:] for x in ip.split('.')]), 2)
def create_hilbert_points(ip_list, order=16):
points = []
for ip in ip_list:
ip_int = ip_to_int(ip)
x, y = hilbert_xy(ip_int, order)
points.append((x, y))
return points
def visualize_hilbert_points(ip_list, order=16, grid_size=32):
points = create_hilbert_points(ip_list, order)
x, y = zip(*points)
fig, ax = plt.subplots(figsize=(12, 12))
# Create density grid
H, xedges, yedges = np.histogram2d(x, y, bins=grid_size, range=[[0, 2**order], [0, 2**order]])
# Plot heatmap
im = ax.imshow(H.T, cmap='YlOrRd', extent=[0, 2**order, 0, 2**order], origin='lower',
norm=colors.LogNorm(vmin=1, vmax=H.max()))
# Plot points
ax.scatter(x, y, color='blue', s=2, alpha=0.5)
ax.set_xlim(0, 2 ** order)
ax.set_ylim(0, 2 ** order)
plt.title(f"IP Addresses on Hilbert Curve (order {order}) with Density Heatmap")
plt.colorbar(im, label='Number of IPs')
plt.axis('off')
plt.show()
if __name__ == '__main__':
ip_list = open(sys.argv[1]).read().splitlines()
visualize_hilbert_points(ip_list)
hilbert_standard_ports.py
-37
View File
@@ -1,37 +0,0 @@
def inverse_hilbert_xy(x, y, order=16):
"""Convert (x, y) on a Hilbert curve of given order to IP integer."""
ip_int = 0
for i in range(order - 1, -1, -1):
xi = (x >> i) & 1
yi = (y >> i) & 1
if yi == 0:
if xi == 1:
x, y = (1 << order) - 1 - y, (1 << order) - 1 - x
x, y = y, x
ip_int |= (xi << (2 * i)) | (yi << (2 * i + 1))
return ip_int
def ip_int_to_ip(ip_int):
"""Convert an integer IP address to a string IP address."""
return ".".join(str((ip_int >> i) & 0xFF) for i in [24, 16, 8, 0])
def generate_ip_ranges(size=16, order=16):
# for x in range(size):
# for y in range(size):
# x_pos = int((x/size)*(2 ** order)/2)
# y_pos = int((y/size)*(2 ** order)/2)
# ip = inverse_hilbert_xy(x_pos, y_pos, order=order)
# print(ip_int_to_ip(ip))
for x in range(256):
for y in range(256):
print(str(x)+"."+str(y)+".255.255")
if __name__ == "__main__":
generate_ip_ranges()
ip_parser.py
+63
View File
@@ -0,0 +1,63 @@
import re
import sys
import utils
def process_ips(file_path, verbose=True):
ip_port_map = {}
ip_regex = r'(?P<ip>(?:\d{1,3}\.){3}\d{1,3})(?::(?P<port>\d{1,5}))?'
ip_count = 0
if verbose:
print("Reading " + file_path)
with open(file_path, 'r') as file:
for line in file:
matches = re.finditer(ip_regex, line)
for match in matches:
ip = match.group('ip')
port = match.group('port')
ip_count += 1
if ip in ip_port_map:
if port:
ip_port_map[ip].add(port)
else:
ip_port_map[ip] = {port} if port else set()
unique_ips = []
for ip, ports in ip_port_map.items():
if ports: # Only add ports if they exist
for port in ports:
if port:
unique_ips.append(f"{ip}:{port}")
else:
unique_ips.append(ip)
else:
unique_ips.append(ip)
del ip_port_map
if verbose:
print("Removed " + str(ip_count - len(unique_ips)) + " Duplicate IPs")
# removed_count = 0
# valid_ips = []
#
# for ip in unique_ips:
# if utils.is_valid_ip(ip):
# valid_ips.append(ip)
# else:
# removed_count += 1
# print(ip)
#
#
# del unique_ips
# print("Removed " + str(removed_count) + " invalid IPs")
if verbose:
print("Retrieved " + str(len(unique_ips)) + " IPs")
return unique_ips
main.py
+43
View File
@@ -0,0 +1,43 @@
import sys
import traceback
import graphs
import ip_parser
order = 16
def printHelp():
print("""
Usage:
./main.py <command> [options]
Commands:
parse <input file>
graph_density <input file>
graph_ports <input file>
graph_nonstandard_ports <input file>
generate_random_ips <input file> <count>
""")
if __name__ == '__main__':
try:
argv = sys.argv
match argv[1]:
case 'parse':
for ip in ip_parser.process_ips(sys.argv[2], verbose=False):
print(ip)
case 'graph_density':
graphs.graph_density(ip_parser.process_ips(sys.argv[2]))
case 'graph_ports':
graphs.graph_ports(ip_parser.process_ips(sys.argv[2]))
case 'graph_nonstandard_ports':
graphs.graph_nonstandard_ports(ip_parser.process_ips(sys.argv[2]))
case 'generate_random_ips':
graphs.generate_random_ips(ip_parser.process_ips(sys.argv[2]), int(sys.argv[3]))
case _:
printHelp()
except Exception as e:
ex_type, ex, tb = sys.exc_info()
traceback.print_tb(tb)
print(str(e))
printHelp()
temp.py
-4
View File
@@ -1,4 +0,0 @@
with open("./ips_1.txt") as f:
lines = f.readlines()
for line in lines:
print(line.split("/")[1].split(":")[0].rstrip())
test.txt
-65536
View File
File diff suppressed because it is too large Load Diff
tor.txt
-8054
View File
File diff suppressed because it is too large Load Diff
utils.py
+93
View File
@@ -0,0 +1,93 @@
# https://github.com/bonsaiviking/IPMap/blob/master/ipmap/hilbert.py
def xy2d(side_length: int, x: int, y: int):
"""Find the distance of the point (x, y) along a Hilbert curve
which fills a square *side_length* units on a side."""
y = side_length-y
s = side_length / 2
d = 0
while( s > 0 ):
rx = 1 if (int(x) & int(s)) else 0
ry = 1 if (int(y) & int(s)) else 0
d += s * s * ((3 * rx) ^ ry)
if (ry == 0):
if (rx == 1):
x = s-1 - x
y = s-1 - y
(x, y) = (y, x)
s /= 2
return int(d)
# https://github.com/bonsaiviking/IPMap/blob/master/ipmap/hilbert.py
def d2xy(side_length: int, d: int):
"""Find the coordinates (x, y) of a point some distance *d* along
a Hilbert curve which fills a square *side_length* units on a side."""
x = y = 0
s = 1
while( s < side_length ):
rx = 1 & int(d/2)
ry = 1 & (int(d) ^ rx)
if (ry == 0):
if (rx == 1):
x = s-1 - x
y = s-1 - y
(x, y) = (y, x)
x += s * rx
y += s * ry
d /= 4
s *= 2
# if y < 10000:
# print(ip_int_to_ip(int(d)))
return x, side_length-y
def ip_to_int(ip):
return int(''.join([bin(int(x) + 256)[3:] for x in ip.split('.')]), 2)
def ip_int_to_ip(ip_int):
return ".".join(str((ip_int >> i) & 0xFF) for i in [24, 16, 8, 0])
def ipInCIDR(ip: str, ip_CIDR: str):
range_parts = ip_CIDR.split('/')
range_mask = int(range_parts[1])
range_mask_num = (0xFFFFFFFF << (32 - range_mask)) & 0xFFFFFFFF
return (ip_to_int(ip) & range_mask_num) == (ip_to_int(range_parts[0]) & range_mask_num)
def ipInRange(ip: str, ip_range: str):
ip_int = ip_to_int(ip)
start_ip_int, end_ip_int = [ip_to_int(ip) for ip in ip_range.split('-')]
return start_ip_int <= ip_int <= end_ip_int
reserved_ip_blocks=[
"0.0.0.0/8",
"10.0.0.0/8",
"100.64.0.0/10",
"127.0.0.0/8",
"169.254.0.0/16",
"172.16.0.0/12",
"192.0.0.0/24",
"192.0.2.0/24",
"192.88.99.0/24",
"192.168.0.0/16",
"198.18.0.0/15",
"198.51.100.0/24",
"203.0.113.0/24",
"224.0.0.0/4",
"233.252.0.0/24",
"240.0.0.0/4",
"255.255.255.255/32"
]
def is_valid_ip(ip: str):
if ":" in ip:
ip = ip.split(":")[0]
for reserved_ip in reserved_ip_blocks:
if ipInCIDR(ip, reserved_ip):
return False
return True