Add Scan Settings page, Work on saving data

TODO:
- Finish saving data

README.md

@@ -5,7 +5,7 @@ Shodan is expensive, IVRE is painful to set up, and there aren't really any othe
 
 Todo:
 - Actually save data
-- Scan settings page
+- ~~Scan settings page~~
 - data on currently ongoing scan
 - Result data Visualization
 - Result data search page

libs/scanner.py

@@ -5,6 +5,10 @@ import importlib
 import socket
 import struct
 import re
+import zlib
+import base64
+import shutil
+import resource
 from threading import Thread
 
 import nmap
@@ -17,6 +21,11 @@ import libs.scanners.udpScanner as udpScanner
 
 portScanners = []
 tasks = []
+excludeRanges = []
+
+downIps = 0
+upIps = 0
+globalSettings = {}
 
 for script in utils.listSubdirs(utils.getRoot("libs/scanners/")):
   if not script.endswith(".py"): continue
@@ -31,37 +40,20 @@ for script in utils.listSubdirs(utils.getRoot("libs/scanners/")):
   print(f'Imported: {utils.getRoot(f"libs/scanners/{module.__name__}")}')
 
   
+soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
+resource.setrlimit(resource.RLIMIT_NOFILE, (hard_limit, hard_limit))
 
-def scan(host:str, port: int, protocol: str):
-  error = False
-  results = ""
-  for scanner in portScanners:
-    if str(scanner.__name__) == f'{protocol}{port}.py':
-      scanResults, error = scanner.scan(host, port)
-      results += f'[{scanner.__name__}, {host}:{port}] {scanResults}'
-      if not error:
-        return results
-      else:
-        results += " Trying default scanner... "
-    
-  if protocol == "tcp":
-    scanResults, error = tcpScanner.scan(host, port)
-    results += f'[{tcpScanner.__name__}, {host}:{port}] {scanResults}'
-  elif protocol == "udp":
-    scanResults, error = udpScanner.scan(host, port)
-    results += f'[{udpScanner.__name__}, {host}:{port}] {scanResults}'
-  else:
-    raise Exception("This should not happen...")
-
-  return results
 
 
 def start(settings):
   global tasks
+  global globalSettings
+  globalSettings = settings
   
   if processStarted(): return
   
   print("\n\nStarted Scanner!")
+  print("\n\n\n", end='')
   utils.makeDir("data/scans")
 
   portString = ""
@@ -89,9 +81,11 @@ def start(settings):
     case 3:
       portString += "T:" + ",".join(map(str, scanutils.portsRelatedTo('tcp', settings['tcpSettings']['relatedString'])))
 
+  global excludeRanges
+  excludeRanges = scanutils.parseIpList(utils.getRoot("exclude.conf"))
 
   for i in range(0,settings['numJobs']):
-    c = ScanTask(i)
+    c = ScanTask(i+1)
     t = Thread(target = c.run, args=(
       settings['maxPingTimeout'],
       settings['maxNmapTimeout'],
@@ -102,6 +96,7 @@ def start(settings):
 
 
 def stop():
+  if not processStarted(): return
   global tasks
   for task in tasks:
     task.stop()
@@ -114,62 +109,171 @@ def processStarted():
   return len(tasks) != 0;
 
 
+class hostScanDetail:
+  def __init__(self):
+    self.address = None
+    self.hostname = None
+    # self.
+
+
+
 
 def parseNmapResult(result: object, host: str):
   
-  hostname = result.hostname()
+  # dict_keys(['hostnames', 'addresses', 'vendor', 'status', 'tcp', 'portused', 'osmatch'])
-  resultstr = f'### {host} ({hostname}) {result.keys()}\n'
   
-  # resultstr += f'Location: {scanutils.geolocation(host)}\n'
+  resultstr = '### Start Host Info ###\n'
   
+  resultstr += f'Address: {host}\n'
+  resultstr += 'Status: Up\n'
+  resultstr += f'Hostname: {result.hostname()}\n'
+  resultstr += f'Location: {scanutils.geolocation(host)}\n'
+
+  osInfo = []
+  if 'osmatch' in result:
+    for os in result['osmatch']:
+      osInfo.append([os["accuracy"], os["name"]])
+
+  resultstr += f'OS-Info: {osInfo}\n'
+
   for protocol in result.all_protocols():
     for portInt in result[protocol].keys():
       port = result[protocol][portInt]
       
-      if port['state'] != 'open':
+      resultstr += f'Port: {portInt},{protocol},{port["state"]},{port["reason"]}'
-        continue
       
-      resultstr += scan(host, portInt, protocol) + "\n"
+      if port['state'] == 'open':
+        data = scan(host, portInt, protocol)
+        compressedData = base64.b64encode(zlib.compress(data.encode())).decode('ASCII')
+        
+        resultstr += f',{compressedData}'
       
-  print(resultstr)
+      resultstr += "\n"
 
+  resultstr += '### End Host Info ###\n'
+      
+  print(resultstr, end='')
+
+def addOfflineHost(host:str):
+  string = '### Start Host Info ###\n' + \
+          f'Address: {host}\n' + \
+          f'Status: Down\n' + \
+           '### End Host Info ###\n'
+  # print(string, end='')
+            
+  
+
+
+def scan(host:str, port: int, protocol: str):
+  error = False
+  results = ""
+  for scanner in portScanners:
+    if str(scanner.__name__) == f'{protocol}{port}.py':
+      scanResults, error = scanner.scan(host, port)
+      results += f'[{scanner.__name__}, {host}:{port}] {scanResults}'
+      if not error:
+        return results
+      else:
+        results += " Trying default scanner... "
+    
+  if protocol == "tcp":
+    scanResults, error = tcpScanner.scan(host, port)
+    results += f'[{tcpScanner.__name__}, {host}:{port}] {scanResults}'
+  elif protocol == "udp":
+    scanResults, error = udpScanner.scan(host, port)
+    results += f'[{udpScanner.__name__}, {host}:{port}] {scanResults}'
+  else:
+    raise Exception("This should not happen...")
+
+  return results
+
+
+# def saveData()
+
+
+def printBar(percentage: float, cols: int):
+  return ("#" * round(percentage*cols)) + ("-" * round((1-percentage) * cols))
+
+
+
+def printIndicator():
+  return
+  hostSearchingCount = 0
+  nmapScanningCount = 0
+  furtherScanningCount = 0
+  for task in tasks:
+    match task.status:
+      case 1:
+        hostSearchingCount += 1
+      case 2:
+        nmapScanningCount += 1
+      case 3:
+        furtherScanningCount += 1
+  
+  width = shutil.get_terminal_size((80, 20)).columns
+  global globalSettings
+  numJobs = int(globalSettings['numJobs'])
+  
+  print("\033[F\033[F\033[F" +
+        f"P: {printBar(hostSearchingCount/numJobs,(width-3))}\n" +
+        f"N: {printBar(nmapScanningCount/numJobs,(width-3))}\n" +
+        f"S: {printBar(furtherScanningCount/numJobs,(width-3))}\n", end="")
+  # print(f"1: {hostSearchingCount}, " +
+  #       f"2: {nmapScanningCount}, " +
+  #       f"3: {furtherScanningCount}", end="\r")
 
 class ScanTask: 
   def __init__(self, threadid: int):
     self.threadid = threadid
     self.running = True
     self.nm = nmap.PortScanner()
+    self.pingsock = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_ICMP)
+    self.status = None
       
   def stop(self): 
-      self.running = False
+    self.running = False
         
   def run(self, maxPingTimeout: int, maxNmapTimeout: int, nmapGroupSize: int, portString: str): 
+    
+    global upIps
+    global downIps
+    global excludeRanges
+    
+
     while self.running:
+      
+      self.status = 1
+      printIndicator()
       ipGroup = []
+      
       while len(ipGroup) < nmapGroupSize and self.running:
         address = socket.inet_ntoa(struct.pack('>I', random.randint(1, 0xffffffff)))
 
-        pingCommand = f"ping {address} -c 1 -W {maxPingTimeout}"
+        if scanutils.ipInArray(address, excludeRanges):
+          # print(f"Tried {address}")
+          continue
 
-        try:
+        if scanutils.ping(address, maxPingTimeout, self.pingsock):
-          subprocess.check_output(pingCommand.split(" "))
+          # print(f"{self.threadid} {address}: FOUND {len(ipGroup)+1}/{nmapGroupSize}")
-          print(f"{self.threadid} {address}: FOUND {len(ipGroup)+1}/{nmapGroupSize}")
+          upIps += 1
           ipGroup.append(address)
-        except subprocess.CalledProcessError:
+        else:
+          addOfflineHost(address)
+          downIps += 1
           # print(f"{self.threadid} {address}: FAIL")
           continue
-      print(f'Scanning: {ipGroup}')
+
+      if not self.running: return
+
+      self.status = 2
+      printIndicator()
 
       self.nm.scan(hosts=' '.join(ipGroup), ports=portString, arguments="-O --send-eth --privileged -sS --reason -sU")
       
+      if not self.running: return
+      
+      self.status = 3
+      printIndicator()
+
       for address in self.nm.all_hosts():
         parseNmapResult(self.nm[address], address)
-
-      # nmapCommand = f"sudo nmap {address} -O --send-eth --privileged -v -sS --reason -sU -p {portString}"
-    
-      
-      # try:
-      #   parseNmapResult(subprocess.check_output(nmapCommand.split(" ")).decode(), address)
-      # except subprocess.CalledProcessError:
-      #   continue
-      

libs/scanutils.py

@@ -5,16 +5,83 @@ import os
 import re
 import geoip2.database
 
-def countScannedIps():
+import socket
-  files = utils.listSubdirs("data/")
+import struct
-  count = 0
+import time
-  for file in files:
+
-    if file.split("-")[0] != "scan":
+def checksum(data):
-      continue
+    """
-    with open('data/'+file) as f:
+    Calculate the checksum of the ICMP packet data.
-      #Count lines in scan files, Masscan has a 2 line header, so hence -2
+    """
-      count += sum(1 for _ in f)-2
+    sum = 0
-  return count
+    for i in range(0, len(data), 2):
+      sum += (data[i] << 8) + data[i+1]
+    sum = (sum & 0xffff) + (sum >> 16)
+    sum = ~sum & 0xffff
+    return sum
+
+import time
+import random
+import select
+import array
+
+
+def ping_chksum(packet:bytes):
+  if len(packet) % 2 != 0:
+    packet += b'\0'
+
+  res = sum(struct.unpack("!%sH" % (len(packet) // 2), packet))
+  res = (res >> 16) + (res & 0xffff)
+  res += res >> 16
+
+  return (~res) & 0xffff
+
+
+def ping(host:str, timeout:int, sock):
+  returnVal = (False, -1)
+  try:
+    # Craft the ICMP echo request packet
+    packet_id = int(time.time() * 1000) & 0xFFFF
+    header = struct.pack("bbHHh", 8, 0, 0, packet_id, 1)
+    data = b"ping"
+    checksum = ping_chksum(header + data)
+    header = struct.pack("bbHHh", 8, 0, socket.htons(checksum), packet_id, 1)
+    packet = header + data
+
+    # Send the ICMP echo request
+    sock.sendto(packet, (host, 1))
+
+    # Receive the ICMP echo reply
+    start_time = time.time()
+    while True:
+      remaining_time = timeout - (time.time() - start_time)
+      if remaining_time <= 0:
+        return False
+      ready = select.select([sock], [], [], remaining_time)
+      if ready[0]:
+        data, addr = sock.recvfrom(1024)
+        icmp_header = data[20:28]
+        type, code, checksum, p_id, sequence = struct.unpack("bbHHh", icmp_header)
+        if p_id == packet_id:
+          return returVal
+
+  except:pass
+  return returnVal
+  #     return False
+
+
+
+
+# def countScannedIps():
+#   files = utils.listSubdirs("data/")
+#   count = 0
+#   for file in files:
+#     if file.split("-")[0] != "scan":
+#       continue
+#     with open('data/'+file) as f:
+#       #Count lines in scan files, Masscan has a 2 line header, so hence -2
+#       count += sum(1 for _ in f)-2
+#   return count
 
 def getMostCommon(protocol: str, n: int):
   nmap_services_file = "/usr/share/nmap/nmap-services"
@@ -125,7 +192,7 @@ if not utils.pathExists(ASN_DB_PATH):
 
 
 
-def geolocation(ip_address):
+def geolocation(ip_address: str):
   try:
     # Attempt to retrieve city-level information
     with geoip2.database.Reader(CITY_DB_PATH) as reader:
@@ -166,3 +233,45 @@ def geolocation(ip_address):
           'ip': ip_address,
           'error': 'No geolocation data found'
         }
+
+def parseIpList(path: str):
+  with open(path, "r") as f:
+    lines = f.readlines()
+    return [line.rstrip() 
+            for line in lines 
+            if not line.startswith('#') and not line.startswith('\n')
+            ]
+
+
+def ipToInt(ip: str):
+  octets = [int(n) for n in ip.split('.')]
+  return (octets[0] << 24) + (octets[1] << 16) + (octets[2] << 8) + octets[3]
+
+
+def ipInCIDR(ip: str, ip_CIDR: str):
+  range_parts = ip_CIDR.split('/')
+  range_mask = int(range_parts[1])
+
+  range_mask_num = (0xFFFFFFFF << (32 - range_mask)) & 0xFFFFFFFF
+
+  return (ipToInt(ip) & range_mask_num) == (ipToInt(range_parts[0]) & range_mask_num)
+
+
+def ipInRange(ip: str, ip_range: str):
+  ip_int = ipToInt(ip)
+  start_ip_int, end_ip_int = [ipToInt(ip) for ip in ip_range.split('-')]
+  
+  return start_ip_int <= ip_int <= end_ip_int
+
+
+def ipInArray(ip: str, ipRangeArray: list):
+  for ipRange in ipRangeArray:
+    if "/" in ipRange:
+      if ipInCIDR(ip, ipRange):
+        return True
+    elif "-" in ipRange:
+      if ipInRange(ip, ipRange):
+        return True
+    elif ip == ipRange:
+      return True
+  return False

modules/main/main.py

@@ -2,9 +2,10 @@ import libs.scanutils as scanutils
 mm = None
 
 def dashboardMetrics(ac):
-  ac.send('Scanner-Metrics', {
+  pass
-    "scanCount": scanutils.countScannedIps()
+  # ac.send('Scanner-Metrics', {
-  })
+    # "scanCount": scanutils.countScannedIps()
+  # })
 
 def init(moduleMaster):
   global mm

modules/scan/main.py

@@ -9,7 +9,46 @@ def loadSettings(ac):
 
 
 def setSettings(ac, data):
-  mm.vars['Scanner-Settings'] = data['data']
+  if not mm.userInGroup(ac, 'Admins'):
+    mm.sendPopupError(ac.rawClient, "Error", "You are not authorised")
+    return
+  
+  data = data['data']
+  valid = True
+  
+  valid = valid and isinstance(data['numJobs'], int)
+  valid = valid and isinstance(data['maxPingTimeout'], int)
+  valid = valid and isinstance(data['maxNmapTimeout'], int)
+  valid = valid and isinstance(data['nmapGroupSize'], int)
+  
+  valid = valid and isinstance(data['tcpSettings'], dict)
+  valid = valid and isinstance(data['udpSettings'], dict)
+  
+  valid = valid and isinstance(data['tcpSettings']['mode'], int)
+  valid = valid and isinstance(data['udpSettings']['mode'], int)
+  
+  if valid:
+    for obj in [data['tcpSettings'], data['udpSettings']]:
+      match obj['mode']:
+        case -1:
+          pass
+        case 1:
+          valid = valid and isinstance(obj['mode'], int)
+          valid = valid and isinstance(obj['ports'], list)
+          if valid:
+            valid = valid and all(isinstance(val, int) for val in obj['ports'])
+        case 2:
+          valid = valid and isinstance(obj['topCount'], int)
+        case 3:
+          valid = valid and isinstance(obj['relatedString'], str)
+        case _:
+          valid = False
+      
+  if valid:
+    print(data)
+    mm.vars['Scanner-Settings'] = data
+  else:
+    mm.sendPopupError(ac.rawClient, "Error", "There is an error in the config.")
 
 
 def startScanner(ac, data):
@@ -26,11 +65,10 @@ def init(moduleMaster):
   mm = moduleMaster
   
   mm.vars['Scanner-Settings'] = {
-    "range": [[0,0,0,0], [255,255,255,255]],
+    "numJobs": 500,
-    "numJobs": 30,
     "maxPingTimeout": 3,
     "maxNmapTimeout": 2,
-    "nmapGroupSize": 3,
+    "nmapGroupSize": 10,
     
     # Port modes:
     # -1: Disable
@@ -40,21 +78,20 @@ def init(moduleMaster):
     
     "tcpSettings": {
       "mode": 2,
-      "ports": [631],
+      "ports": [22, 80, 443],
-      "topCount": 10,
+      "topCount": 100,
       "relatedString": "http"
     },
     "udpSettings": {
       "mode": -1,
-      "ports": [631, 161, 137, 123, 138],
+      "ports": [631, 161, 137],
-      "topCount": 50,
+      "topCount": 100,
       "relatedString": "telnet"
-    },
+    }
-    "runTCP": True,
-    "runUDP": False
   } 
   
-  mm.addPageEventListener('Scanner-LoadSettings', loadSettings)
+  mm.addPageEventListener('/Scan/Scan', loadSettings)
+  mm.addAuthEventListener('Scanner-SetSettings', setSettings)
   
   mm.addAuthEventListener('Scanner-StartScanner', startScanner)
   mm.addAuthEventListener('Scanner-StopScanner', stopScanner)

modules/scan/scan.html

@@ -1,16 +1,168 @@
 
 <main class="container">
-  <h4>This is a very simple example module!</h4>
+  <h2>Scan Settings</h2>
   <button class="half-left" onclick="startScanner()">Start Scanner</button>
   <button class="half-right" onclick="stopScanner()">Stop Scanner</button>
+  <button onclick="saveConfig()">Save Config</button>
+
+  <br>
+  <br>
+
+  <p>Number of scanning threads</p>
+  <input id="settingsNumJobs" type="number" />
+  <p>Ping command timeout (seconds)</p>
+  <input id="settingsPingTimeout" type="number" />
+  <p>Nmap command timeout (seconds)</p>
+  <input id="settingsNmapTimeout" type="number" />
+  <p>Nmap accumulated host count</p>
+  <input id="settingsNmapGroupSize" type="number" />
+
+  <p>TCP Settings</p>
+  <select id="settingsTCPDropdown" onchange="updateTCPDropdown()">
+    <option selected value=-1>Disable</option>
+    <option value=1>Specify ports</option>
+    <option value=2>N Most common ports</option>
+    <option value=3>Ports related to string</option>
+  </select>
+  <input id="settingsTCP" style="display: none;" />
+
+  <p>UDP Settings</p>
+  <select id="settingsUDPDropdown" onchange="updateUDPDropdown()">
+    <option selected value=-1>Disable</option>
+    <option value=1>Specify ports</option>
+    <option value=2>N Most common ports</option>
+    <option value=3>Ports related to string</option>
+  </select>
+  <input id="settingsUDP" style="display: none;" />
+
 </main>
 
 <script>
-  window.main = ()=>{
+  function getel(el) {return document.getElementById(el)}
-    window.addListener('Scanner-LoadSettings', (data)=>{})
+
+  const settingsNumJobs = getel('settingsNumJobs')
+  const settingsPingTimeout = getel('settingsPingTimeout')
+  const settingsNmapTimeout = getel('settingsNmapTimeout')
+  const settingsNmapGroupSize = getel('settingsNmapGroupSize')
+  const settingsTCPDropdown = getel('settingsTCPDropdown')
+  const settingsTCP = getel('settingsTCP')
+  const settingsUDPDropdown = getel('settingsUDPDropdown')
+  const settingsUDP = getel('settingsUDP')
+
+  let settings = {
+    "numJobs": -1,
+    "maxPingTimeout": -1,
+    "maxNmapTimeout": -1,
+    "nmapGroupSize": -1,
+    
+    "tcpSettings": {
+      "mode": -1,
+      "ports": [],
+      "topCount": -1,
+      "relatedString": ""
+    },
+    "udpSettings": {
+      "mode": -1,
+      "ports": [],
+      "topCount": -1,
+      "relatedString": ""
+    }
+  } 
+
+  function updateTCPDropdown(){
+    if(settingsTCPDropdown.value == -1){
+      settingsTCP.style.display = "none"
+    }else{
+      settingsTCP.style.display = ""
+      switch(Number(settingsTCPDropdown.value)){
+        case 1:
+          settingsTCP.value = settings.tcpSettings.ports.join(", ")
+          break;
+        case 2:
+          settingsTCP.value = settings.tcpSettings.topCount
+          break;
+        case 3:
+          settingsTCP.value = settings.tcpSettings.relatedString
+          break;
+      }
+    }
   }
 
-  function getel(el) {return document.getElementById(el)}
+  function updateUDPDropdown(){
+    if(settingsUDPDropdown.value == -1){
+      settingsUDP.style.display = "none"
+    }else{
+      settingsUDP.style.display = ""
+      switch(Number(settingsUDPDropdown.value)){
+        case 1:
+          settingsUDP.value = settings.udpSettings.ports.join(", ")
+          break;
+        case 2:
+          settingsUDP.value = settings.udpSettings.topCount
+          break;
+        case 3:
+          settingsUDP.value = settings.udpSettings.relatedString
+          break;
+      }
+    }
+  }
+
+  window.main = ()=>{
+
+
+
+  window.addListener('Scanner-Settings', (data)=>{
+    settings = data.data
+    settingsNumJobs.value = settings.numJobs
+    settingsPingTimeout.value = settings.maxPingTimeout
+    settingsNmapTimeout.value = settings.maxNmapTimeout
+    settingsNmapGroupSize.value = settings.nmapGroupSize
+    settingsTCPDropdown.value = settings.tcpSettings.mode
+    settingsUDPDropdown.value = settings.udpSettings.mode
+
+    updateTCPDropdown()
+    updateUDPDropdown()
+  })
+
+  window.client.send('Scanner-LoadSettings', {})
+
+  }
+
+  function saveConfig() {
+    settings.numJobs = Number(settingsNumJobs.value)
+    settings.maxPingTimeout = Number(settingsPingTimeout.value)
+    settings.maxNmapTimeout = Number(settingsNmapTimeout.value)
+    settings.nmapGroupSize = Number(settingsNmapGroupSize.value)
+    settings.tcpSettings.mode = Number(settingsTCPDropdown.value)
+    settings.udpSettings.mode = Number(settingsUDPDropdown.value)
+
+    switch(Number(settingsTCPDropdown.value)){
+      case 1:
+      settings.tcpSettings.ports = settingsTCP.value.split(',').map(Number)
+        break;
+      case 2:
+        settings.tcpSettings.topCount = Number(settingsTCP.value)
+        break;
+      case 3:
+        settings.tcpSettings.relatedString = settingsTCP.value
+        break;
+    }
+
+    switch(Number(settingsUDPDropdown.value)){
+      case 1:
+        settings.udpSettings.ports = settingsUDP.value.split(',').map(Number)
+        break;
+      case 2:
+        settings.udpSettings.topCount = Number(settingsUDP.value)
+        break;
+      case 3:
+        settings.udpSettings.relatedString = settingsUDP.value
+        break;
+    }
+
+    // console.log(settings)
+    window.send("Scanner-SetSettings", settings)
+  }
 
   function startScanner() {
     window.send('Scanner-StartScanner', {})

requirements.txt

@@ -2,3 +2,4 @@ flask
 geoip2
 faker
 python-nmap
+pyopenssl