Add Scan Settings page, Work on saving data

TODO:
- Finish saving data
This commit is contained in:
astatin3
2024-04-19 23:11:46 -06:00
parent 96285a4c58
commit c21af54bcd
7 changed files with 481 additions and 77 deletions
+1 -1
View File
@@ -5,7 +5,7 @@ Shodan is expensive, IVRE is painful to set up, and there aren't really any othe
Todo:
- Actually save data
- Scan settings page
- ~~Scan settings page~~
- data on currently ongoing scan
- Result data Visualization
- Result data search page
+150 -46
View File
@@ -5,6 +5,10 @@ import importlib
import socket
import struct
import re
import zlib
import base64
import shutil
import resource
from threading import Thread
import nmap
@@ -17,6 +21,11 @@ import libs.scanners.udpScanner as udpScanner
portScanners = []
tasks = []
excludeRanges = []
downIps = 0
upIps = 0
globalSettings = {}
for script in utils.listSubdirs(utils.getRoot("libs/scanners/")):
if not script.endswith(".py"): continue
@@ -31,37 +40,20 @@ for script in utils.listSubdirs(utils.getRoot("libs/scanners/")):
print(f'Imported: {utils.getRoot(f"libs/scanners/{module.__name__}")}')
soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE, (hard_limit, hard_limit))
def scan(host:str, port: int, protocol: str):
error = False
results = ""
for scanner in portScanners:
if str(scanner.__name__) == f'{protocol}{port}.py':
scanResults, error = scanner.scan(host, port)
results += f'[{scanner.__name__}, {host}:{port}] {scanResults}'
if not error:
return results
else:
results += " Trying default scanner... "
if protocol == "tcp":
scanResults, error = tcpScanner.scan(host, port)
results += f'[{tcpScanner.__name__}, {host}:{port}] {scanResults}'
elif protocol == "udp":
scanResults, error = udpScanner.scan(host, port)
results += f'[{udpScanner.__name__}, {host}:{port}] {scanResults}'
else:
raise Exception("This should not happen...")
return results
def start(settings):
global tasks
global globalSettings
globalSettings = settings
if processStarted(): return
print("\n\nStarted Scanner!")
print("\n\n\n", end='')
utils.makeDir("data/scans")
portString = ""
@@ -89,9 +81,11 @@ def start(settings):
case 3:
portString += "T:" + ",".join(map(str, scanutils.portsRelatedTo('tcp', settings['tcpSettings']['relatedString'])))
global excludeRanges
excludeRanges = scanutils.parseIpList(utils.getRoot("exclude.conf"))
for i in range(0,settings['numJobs']):
c = ScanTask(i)
c = ScanTask(i+1)
t = Thread(target = c.run, args=(
settings['maxPingTimeout'],
settings['maxNmapTimeout'],
@@ -102,6 +96,7 @@ def start(settings):
def stop():
if not processStarted(): return
global tasks
for task in tasks:
task.stop()
@@ -114,62 +109,171 @@ def processStarted():
return len(tasks) != 0;
class hostScanDetail:
def __init__(self):
self.address = None
self.hostname = None
# self.
def parseNmapResult(result: object, host: str):
hostname = result.hostname()
resultstr = f'### {host} ({hostname}) {result.keys()}\n'
# dict_keys(['hostnames', 'addresses', 'vendor', 'status', 'tcp', 'portused', 'osmatch'])
# resultstr += f'Location: {scanutils.geolocation(host)}\n'
resultstr = '### Start Host Info ###\n'
resultstr += f'Address: {host}\n'
resultstr += 'Status: Up\n'
resultstr += f'Hostname: {result.hostname()}\n'
resultstr += f'Location: {scanutils.geolocation(host)}\n'
osInfo = []
if 'osmatch' in result:
for os in result['osmatch']:
osInfo.append([os["accuracy"], os["name"]])
resultstr += f'OS-Info: {osInfo}\n'
for protocol in result.all_protocols():
for portInt in result[protocol].keys():
port = result[protocol][portInt]
if port['state'] != 'open':
continue
resultstr += f'Port: {portInt},{protocol},{port["state"]},{port["reason"]}'
resultstr += scan(host, portInt, protocol) + "\n"
if port['state'] == 'open':
data = scan(host, portInt, protocol)
compressedData = base64.b64encode(zlib.compress(data.encode())).decode('ASCII')
resultstr += f',{compressedData}'
print(resultstr)
resultstr += "\n"
resultstr += '### End Host Info ###\n'
print(resultstr, end='')
def addOfflineHost(host:str):
string = '### Start Host Info ###\n' + \
f'Address: {host}\n' + \
f'Status: Down\n' + \
'### End Host Info ###\n'
# print(string, end='')
def scan(host:str, port: int, protocol: str):
error = False
results = ""
for scanner in portScanners:
if str(scanner.__name__) == f'{protocol}{port}.py':
scanResults, error = scanner.scan(host, port)
results += f'[{scanner.__name__}, {host}:{port}] {scanResults}'
if not error:
return results
else:
results += " Trying default scanner... "
if protocol == "tcp":
scanResults, error = tcpScanner.scan(host, port)
results += f'[{tcpScanner.__name__}, {host}:{port}] {scanResults}'
elif protocol == "udp":
scanResults, error = udpScanner.scan(host, port)
results += f'[{udpScanner.__name__}, {host}:{port}] {scanResults}'
else:
raise Exception("This should not happen...")
return results
# def saveData()
def printBar(percentage: float, cols: int):
return ("#" * round(percentage*cols)) + ("-" * round((1-percentage) * cols))
def printIndicator():
return
hostSearchingCount = 0
nmapScanningCount = 0
furtherScanningCount = 0
for task in tasks:
match task.status:
case 1:
hostSearchingCount += 1
case 2:
nmapScanningCount += 1
case 3:
furtherScanningCount += 1
width = shutil.get_terminal_size((80, 20)).columns
global globalSettings
numJobs = int(globalSettings['numJobs'])
print("\033[F\033[F\033[F" +
f"P: {printBar(hostSearchingCount/numJobs,(width-3))}\n" +
f"N: {printBar(nmapScanningCount/numJobs,(width-3))}\n" +
f"S: {printBar(furtherScanningCount/numJobs,(width-3))}\n", end="")
# print(f"1: {hostSearchingCount}, " +
# f"2: {nmapScanningCount}, " +
# f"3: {furtherScanningCount}", end="\r")
class ScanTask:
def __init__(self, threadid: int):
self.threadid = threadid
self.running = True
self.nm = nmap.PortScanner()
self.pingsock = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_ICMP)
self.status = None
def stop(self):
self.running = False
self.running = False
def run(self, maxPingTimeout: int, maxNmapTimeout: int, nmapGroupSize: int, portString: str):
global upIps
global downIps
global excludeRanges
while self.running:
self.status = 1
printIndicator()
ipGroup = []
while len(ipGroup) < nmapGroupSize and self.running:
address = socket.inet_ntoa(struct.pack('>I', random.randint(1, 0xffffffff)))
pingCommand = f"ping {address} -c 1 -W {maxPingTimeout}"
if scanutils.ipInArray(address, excludeRanges):
# print(f"Tried {address}")
continue
try:
subprocess.check_output(pingCommand.split(" "))
print(f"{self.threadid} {address}: FOUND {len(ipGroup)+1}/{nmapGroupSize}")
if scanutils.ping(address, maxPingTimeout, self.pingsock):
# print(f"{self.threadid} {address}: FOUND {len(ipGroup)+1}/{nmapGroupSize}")
upIps += 1
ipGroup.append(address)
except subprocess.CalledProcessError:
else:
addOfflineHost(address)
downIps += 1
# print(f"{self.threadid} {address}: FAIL")
continue
print(f'Scanning: {ipGroup}')
if not self.running: return
self.status = 2
printIndicator()
self.nm.scan(hosts=' '.join(ipGroup), ports=portString, arguments="-O --send-eth --privileged -sS --reason -sU")
if not self.running: return
self.status = 3
printIndicator()
for address in self.nm.all_hosts():
parseNmapResult(self.nm[address], address)
# nmapCommand = f"sudo nmap {address} -O --send-eth --privileged -v -sS --reason -sU -p {portString}"
# try:
# parseNmapResult(subprocess.check_output(nmapCommand.split(" ")).decode(), address)
# except subprocess.CalledProcessError:
# continue
+120 -11
View File
@@ -5,16 +5,83 @@ import os
import re
import geoip2.database
def countScannedIps():
files = utils.listSubdirs("data/")
count = 0
for file in files:
if file.split("-")[0] != "scan":
continue
with open('data/'+file) as f:
#Count lines in scan files, Masscan has a 2 line header, so hence -2
count += sum(1 for _ in f)-2
return count
import socket
import struct
import time
def checksum(data):
"""
Calculate the checksum of the ICMP packet data.
"""
sum = 0
for i in range(0, len(data), 2):
sum += (data[i] << 8) + data[i+1]
sum = (sum & 0xffff) + (sum >> 16)
sum = ~sum & 0xffff
return sum
import time
import random
import select
import array
def ping_chksum(packet:bytes):
if len(packet) % 2 != 0:
packet += b'\0'
res = sum(struct.unpack("!%sH" % (len(packet) // 2), packet))
res = (res >> 16) + (res & 0xffff)
res += res >> 16
return (~res) & 0xffff
def ping(host:str, timeout:int, sock):
returnVal = (False, -1)
try:
# Craft the ICMP echo request packet
packet_id = int(time.time() * 1000) & 0xFFFF
header = struct.pack("bbHHh", 8, 0, 0, packet_id, 1)
data = b"ping"
checksum = ping_chksum(header + data)
header = struct.pack("bbHHh", 8, 0, socket.htons(checksum), packet_id, 1)
packet = header + data
# Send the ICMP echo request
sock.sendto(packet, (host, 1))
# Receive the ICMP echo reply
start_time = time.time()
while True:
remaining_time = timeout - (time.time() - start_time)
if remaining_time <= 0:
return False
ready = select.select([sock], [], [], remaining_time)
if ready[0]:
data, addr = sock.recvfrom(1024)
icmp_header = data[20:28]
type, code, checksum, p_id, sequence = struct.unpack("bbHHh", icmp_header)
if p_id == packet_id:
return returVal
except:pass
return returnVal
# return False
# def countScannedIps():
# files = utils.listSubdirs("data/")
# count = 0
# for file in files:
# if file.split("-")[0] != "scan":
# continue
# with open('data/'+file) as f:
# #Count lines in scan files, Masscan has a 2 line header, so hence -2
# count += sum(1 for _ in f)-2
# return count
def getMostCommon(protocol: str, n: int):
nmap_services_file = "/usr/share/nmap/nmap-services"
@@ -125,7 +192,7 @@ if not utils.pathExists(ASN_DB_PATH):
def geolocation(ip_address):
def geolocation(ip_address: str):
try:
# Attempt to retrieve city-level information
with geoip2.database.Reader(CITY_DB_PATH) as reader:
@@ -166,3 +233,45 @@ def geolocation(ip_address):
'ip': ip_address,
'error': 'No geolocation data found'
}
def parseIpList(path: str):
with open(path, "r") as f:
lines = f.readlines()
return [line.rstrip()
for line in lines
if not line.startswith('#') and not line.startswith('\n')
]
def ipToInt(ip: str):
octets = [int(n) for n in ip.split('.')]
return (octets[0] << 24) + (octets[1] << 16) + (octets[2] << 8) + octets[3]
def ipInCIDR(ip: str, ip_CIDR: str):
range_parts = ip_CIDR.split('/')
range_mask = int(range_parts[1])
range_mask_num = (0xFFFFFFFF << (32 - range_mask)) & 0xFFFFFFFF
return (ipToInt(ip) & range_mask_num) == (ipToInt(range_parts[0]) & range_mask_num)
def ipInRange(ip: str, ip_range: str):
ip_int = ipToInt(ip)
start_ip_int, end_ip_int = [ipToInt(ip) for ip in ip_range.split('-')]
return start_ip_int <= ip_int <= end_ip_int
def ipInArray(ip: str, ipRangeArray: list):
for ipRange in ipRangeArray:
if "/" in ipRange:
if ipInCIDR(ip, ipRange):
return True
elif "-" in ipRange:
if ipInRange(ip, ipRange):
return True
elif ip == ipRange:
return True
return False
+4 -3
View File
@@ -2,9 +2,10 @@ import libs.scanutils as scanutils
mm = None
def dashboardMetrics(ac):
ac.send('Scanner-Metrics', {
"scanCount": scanutils.countScannedIps()
})
pass
# ac.send('Scanner-Metrics', {
# "scanCount": scanutils.countScannedIps()
# })
def init(moduleMaster):
global mm
+49 -12
View File
@@ -9,7 +9,46 @@ def loadSettings(ac):
def setSettings(ac, data):
mm.vars['Scanner-Settings'] = data['data']
if not mm.userInGroup(ac, 'Admins'):
mm.sendPopupError(ac.rawClient, "Error", "You are not authorised")
return
data = data['data']
valid = True
valid = valid and isinstance(data['numJobs'], int)
valid = valid and isinstance(data['maxPingTimeout'], int)
valid = valid and isinstance(data['maxNmapTimeout'], int)
valid = valid and isinstance(data['nmapGroupSize'], int)
valid = valid and isinstance(data['tcpSettings'], dict)
valid = valid and isinstance(data['udpSettings'], dict)
valid = valid and isinstance(data['tcpSettings']['mode'], int)
valid = valid and isinstance(data['udpSettings']['mode'], int)
if valid:
for obj in [data['tcpSettings'], data['udpSettings']]:
match obj['mode']:
case -1:
pass
case 1:
valid = valid and isinstance(obj['mode'], int)
valid = valid and isinstance(obj['ports'], list)
if valid:
valid = valid and all(isinstance(val, int) for val in obj['ports'])
case 2:
valid = valid and isinstance(obj['topCount'], int)
case 3:
valid = valid and isinstance(obj['relatedString'], str)
case _:
valid = False
if valid:
print(data)
mm.vars['Scanner-Settings'] = data
else:
mm.sendPopupError(ac.rawClient, "Error", "There is an error in the config.")
def startScanner(ac, data):
@@ -26,11 +65,10 @@ def init(moduleMaster):
mm = moduleMaster
mm.vars['Scanner-Settings'] = {
"range": [[0,0,0,0], [255,255,255,255]],
"numJobs": 30,
"numJobs": 500,
"maxPingTimeout": 3,
"maxNmapTimeout": 2,
"nmapGroupSize": 3,
"nmapGroupSize": 10,
# Port modes:
# -1: Disable
@@ -40,21 +78,20 @@ def init(moduleMaster):
"tcpSettings": {
"mode": 2,
"ports": [631],
"topCount": 10,
"ports": [22, 80, 443],
"topCount": 100,
"relatedString": "http"
},
"udpSettings": {
"mode": -1,
"ports": [631, 161, 137, 123, 138],
"topCount": 50,
"ports": [631, 161, 137],
"topCount": 100,
"relatedString": "telnet"
},
"runTCP": True,
"runUDP": False
}
}
mm.addPageEventListener('Scanner-LoadSettings', loadSettings)
mm.addPageEventListener('/Scan/Scan', loadSettings)
mm.addAuthEventListener('Scanner-SetSettings', setSettings)
mm.addAuthEventListener('Scanner-StartScanner', startScanner)
mm.addAuthEventListener('Scanner-StopScanner', stopScanner)
+156 -4
View File
@@ -1,16 +1,168 @@
<main class="container">
<h4>This is a very simple example module!</h4>
<h2>Scan Settings</h2>
<button class="half-left" onclick="startScanner()">Start Scanner</button>
<button class="half-right" onclick="stopScanner()">Stop Scanner</button>
<button onclick="saveConfig()">Save Config</button>
<br>
<br>
<p>Number of scanning threads</p>
<input id="settingsNumJobs" type="number" />
<p>Ping command timeout (seconds)</p>
<input id="settingsPingTimeout" type="number" />
<p>Nmap command timeout (seconds)</p>
<input id="settingsNmapTimeout" type="number" />
<p>Nmap accumulated host count</p>
<input id="settingsNmapGroupSize" type="number" />
<p>TCP Settings</p>
<select id="settingsTCPDropdown" onchange="updateTCPDropdown()">
<option selected value=-1>Disable</option>
<option value=1>Specify ports</option>
<option value=2>N Most common ports</option>
<option value=3>Ports related to string</option>
</select>
<input id="settingsTCP" style="display: none;" />
<p>UDP Settings</p>
<select id="settingsUDPDropdown" onchange="updateUDPDropdown()">
<option selected value=-1>Disable</option>
<option value=1>Specify ports</option>
<option value=2>N Most common ports</option>
<option value=3>Ports related to string</option>
</select>
<input id="settingsUDP" style="display: none;" />
</main>
<script>
window.main = ()=>{
window.addListener('Scanner-LoadSettings', (data)=>{})
function getel(el) {return document.getElementById(el)}
const settingsNumJobs = getel('settingsNumJobs')
const settingsPingTimeout = getel('settingsPingTimeout')
const settingsNmapTimeout = getel('settingsNmapTimeout')
const settingsNmapGroupSize = getel('settingsNmapGroupSize')
const settingsTCPDropdown = getel('settingsTCPDropdown')
const settingsTCP = getel('settingsTCP')
const settingsUDPDropdown = getel('settingsUDPDropdown')
const settingsUDP = getel('settingsUDP')
let settings = {
"numJobs": -1,
"maxPingTimeout": -1,
"maxNmapTimeout": -1,
"nmapGroupSize": -1,
"tcpSettings": {
"mode": -1,
"ports": [],
"topCount": -1,
"relatedString": ""
},
"udpSettings": {
"mode": -1,
"ports": [],
"topCount": -1,
"relatedString": ""
}
}
function updateTCPDropdown(){
if(settingsTCPDropdown.value == -1){
settingsTCP.style.display = "none"
}else{
settingsTCP.style.display = ""
switch(Number(settingsTCPDropdown.value)){
case 1:
settingsTCP.value = settings.tcpSettings.ports.join(", ")
break;
case 2:
settingsTCP.value = settings.tcpSettings.topCount
break;
case 3:
settingsTCP.value = settings.tcpSettings.relatedString
break;
}
}
}
function getel(el) {return document.getElementById(el)}
function updateUDPDropdown(){
if(settingsUDPDropdown.value == -1){
settingsUDP.style.display = "none"
}else{
settingsUDP.style.display = ""
switch(Number(settingsUDPDropdown.value)){
case 1:
settingsUDP.value = settings.udpSettings.ports.join(", ")
break;
case 2:
settingsUDP.value = settings.udpSettings.topCount
break;
case 3:
settingsUDP.value = settings.udpSettings.relatedString
break;
}
}
}
window.main = ()=>{
window.addListener('Scanner-Settings', (data)=>{
settings = data.data
settingsNumJobs.value = settings.numJobs
settingsPingTimeout.value = settings.maxPingTimeout
settingsNmapTimeout.value = settings.maxNmapTimeout
settingsNmapGroupSize.value = settings.nmapGroupSize
settingsTCPDropdown.value = settings.tcpSettings.mode
settingsUDPDropdown.value = settings.udpSettings.mode
updateTCPDropdown()
updateUDPDropdown()
})
window.client.send('Scanner-LoadSettings', {})
}
function saveConfig() {
settings.numJobs = Number(settingsNumJobs.value)
settings.maxPingTimeout = Number(settingsPingTimeout.value)
settings.maxNmapTimeout = Number(settingsNmapTimeout.value)
settings.nmapGroupSize = Number(settingsNmapGroupSize.value)
settings.tcpSettings.mode = Number(settingsTCPDropdown.value)
settings.udpSettings.mode = Number(settingsUDPDropdown.value)
switch(Number(settingsTCPDropdown.value)){
case 1:
settings.tcpSettings.ports = settingsTCP.value.split(',').map(Number)
break;
case 2:
settings.tcpSettings.topCount = Number(settingsTCP.value)
break;
case 3:
settings.tcpSettings.relatedString = settingsTCP.value
break;
}
switch(Number(settingsUDPDropdown.value)){
case 1:
settings.udpSettings.ports = settingsUDP.value.split(',').map(Number)
break;
case 2:
settings.udpSettings.topCount = Number(settingsUDP.value)
break;
case 3:
settings.udpSettings.relatedString = settingsUDP.value
break;
}
// console.log(settings)
window.send("Scanner-SetSettings", settings)
}
function startScanner() {
window.send('Scanner-StartScanner', {})
+1
View File
@@ -2,3 +2,4 @@ flask
geoip2
faker
python-nmap
pyopenssl