Add graphing

libs/scanner.py

@@ -9,7 +9,7 @@ import zlib
 import base64
 import shutil
 import resource
-from threading import Thread
+from threading import Thread, Timer
 
 import nmap
 
@@ -24,9 +24,20 @@ portScanners = []
 tasks = []
 excludeRanges = []
 
-downIps = 0
 upIps = 0
+downIps = 0
+extendedIps = 0
+
+upIpsPS = 0
+downIpsPS = 0
+extendedIpsPS = 0
+
+countScannedBeforeStart = 0
+
+running = False
 globalSettings = {}
+onStatsFunc = None
+
 
 for script in utils.listSubdirs(utils.getRoot("libs/scanners/")):
   if not script.endswith(".py"): continue
@@ -46,12 +57,31 @@ resource.setrlimit(resource.RLIMIT_NOFILE, (hard_limit, hard_limit))
 
 
 
-def start(settings):
+def start(settings, onStats):
+  if processStarted(): return
   global tasks
   global globalSettings
   globalSettings = settings
   
-  if processStarted(): return
+  global onStatsFunc
+  onStatsFunc = onStats
+  
+  global upIps
+  global downIps
+  global extendedIps
+  upIps = 0
+  downIps = 0
+  extendedIps = 0
+  
+  global upIpsPS
+  global downIpsPS
+  global extendedIpsPS
+  upIpsPS = 0
+  downIpsPS = 0
+  extendedIpsPS = 0
+  
+  global countScannedBeforeStart
+  countScannedBeforeStart = scanutils.countScannedIps()
   
   print("\n\nStarted Scanner!")
   print("\n\n\n", end='')
@@ -85,6 +115,9 @@ def start(settings):
   global excludeRanges
   excludeRanges = scanutils.parseIpList(utils.getRoot("exclude.conf"))
 
+  global running
+  running = True
+
   for i in range(0,settings['numJobs']):
     c = ScanTask(i+1)
     t = Thread(target = c.run, args=(
@@ -94,14 +127,16 @@ def start(settings):
       portString,))
     t.start()
     tasks.append(c)
+    
+  statsTimer()
 
 
 def stop():
   if not processStarted(): return
   global tasks
-  for task in tasks:
+  global running
-    task.stop()
   tasks = []
+  running = False
   print("\n\nstopped Scanner!")
 
 
@@ -116,6 +151,54 @@ class hostScanDetail:
     self.hostname = None
     # self.
 
+def statsTimer():
+  if running:
+    Timer(1.0, statsTimer).start()
+
+  global upIps
+  global downIps
+  global extendedIps
+  
+  global upIpsPS
+  global downIpsPS
+  global extendedIpsPS
+  global numJobs
+  global countScannedBeforeStart
+
+  hostSearchingCount = 0
+  nmapScanningCount = 0
+  furtherScanningCount = 0
+  for task in tasks:
+    match task.status:
+      case 1:
+        hostSearchingCount += 1
+      case 2:
+        nmapScanningCount += 1
+      case 3:
+        furtherScanningCount += 1    
+
+  stats = {
+    "upIps": upIps,
+    "downIps": downIps,
+    "extendedIps": extendedIps,
+    
+    "upIpsPS": upIpsPS,
+    "downIpsPS": downIpsPS,
+    "extendedIpsPS": extendedIpsPS,
+    
+    "hostSearchingCount": hostSearchingCount,
+    "nmapScanningCount": nmapScanningCount,
+    "furtherScanningCount": furtherScanningCount,
+    "countScannedBeforeStart": countScannedBeforeStart,
+    "numJobs": int(globalSettings['numJobs'])
+    
+  }
+  upIpsPS = 0
+  downIpsPS = 0
+  extendedIpsPS = 0
+
+  
+  onStatsFunc(stats)
 
 
 
@@ -156,14 +239,20 @@ def parseNmapResult(result: object, host: str):
       if port['state'] == 'open':
         data = scan(host, portInt, protocol)
         compressedData = base64.b64encode(zlib.compress(data.encode())).decode('ASCII')
-        
+
         resultstr += f',{compressedData}'
       
       resultstr += "]"
       
   resultstr += "]"
-      
+  
-  print(resultstr)
+  if len(result.all_protocols()) > 0:
+    global extendedIps
+    global extendedIpsPS
+    extendedIps += 1
+    extendedIpsPS += 1
+  
+  # print(resultstr)
       
   write(host, resultstr)
   
@@ -215,57 +304,35 @@ def printBar(percentage: float, cols: int):
   return ("#" * round(percentage*cols)) + ("-" * round((1-percentage) * cols))
 
 
-
-def printIndicator():
-  hostSearchingCount = 0
-  nmapScanningCount = 0
-  furtherScanningCount = 0
-  for task in tasks:
-    match task.status:
-      case 1:
-        hostSearchingCount += 1
-      case 2:
-        nmapScanningCount += 1
-      case 3:
-        furtherScanningCount += 1
-  
-  width = shutil.get_terminal_size((80, 20)).columns
-  global globalSettings
-  numJobs = int(globalSettings['numJobs'])
-  
-  print("\033[F\033[F\033[F" +
-        f"P: {printBar(hostSearchingCount/numJobs,(width-3))}\n" +
-        f"N: {printBar(nmapScanningCount/numJobs,(width-3))}\n" +
-        f"S: {printBar(furtherScanningCount/numJobs,(width-3))}\n", end="")
-  # print(f"1: {hostSearchingCount}, " +
-  #       f"2: {nmapScanningCount}, " +
-  #       f"3: {furtherScanningCount}", end="\r")
-
 class ScanTask: 
   def __init__(self, threadid: int):
     self.threadid = threadid
-    self.running = True
     self.nm = nmap.PortScanner()
     self.pingsock = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_ICMP)
     self.status = None
-      
-  def stop(self): 
-    self.running = False
         
   def run(self, maxPingTimeout: int, maxNmapTimeout: int, nmapGroupSize: int, portString: str): 
     
     global upIps
     global downIps
+    global extendedIps
+    
+    global upIpsPS
+    global downIpsPS
+    global extendedIpsPS
+    
     global excludeRanges
+    global running
+    
+    
     
 
-    while self.running:
+    while running:
       
       self.status = 1
-      printIndicator()
       ipGroup = []
       
-      while len(ipGroup) < nmapGroupSize and self.running:
+      while len(ipGroup) < nmapGroupSize and running:
         address = socket.inet_ntoa(struct.pack('>I', random.randint(1, 0xffffffff)))
 
         if scanutils.ipInArray(address, excludeRanges):
@@ -275,24 +342,24 @@ class ScanTask:
         if scanutils.ping(address, maxPingTimeout, self.pingsock):
           # print(f"{self.threadid} {address}: FOUND {len(ipGroup)+1}/{nmapGroupSize}")
           upIps += 1
+          upIpsPS += 1
           ipGroup.append(address)
         else:
           addOfflineHost(address)
           downIps += 1
+          downIpsPS += 1
           # print(f"{self.threadid} {address}: FAIL")
           continue
 
-      if not self.running: return
+      if not running: return
 
       self.status = 2
-      printIndicator()
 
       self.nm.scan(hosts=' '.join(ipGroup), ports=portString, arguments="-O --send-eth --privileged -sS --reason -sU")
       
-      if not self.running: return
+      if not running: return
       
       self.status = 3
-      printIndicator()
 
       for address in self.nm.all_hosts():
         parseNmapResult(self.nm[address], address)

libs/scanutils.py

@@ -72,16 +72,73 @@ def ping(host:str, timeout:int, sock):
 
 
 
-# def countScannedIps():
+def countScannedIps():
-#   files = utils.listSubdirs("data/")
+  files = utils.listSubdirs("data/")
-#   count = 0
+  count = 0
-#   for file in files:
+  for folder in utils.listSubdirs(utils.getRoot("data/scans/")):
-#     if file.split("-")[0] != "scan":
+    for file in utils.listSubdirs(utils.getRoot(f"data/scans/{folder}")):
-#       continue
+      with open(utils.getRoot(f"data/scans/{folder}/{file}"), "r") as file:
-#     with open('data/'+file) as f:
+        lines = file.readlines()
-#       #Count lines in scan files, Masscan has a 2 line header, so hence -2
+        count += sum(1 for _ in lines) - 1
-#       count += sum(1 for _ in f)-2
+  return count
-#   return count
+
+
+
+
+def parse_csv_line(line):
+    elements = []
+    current_element = ""
+    inside_array = False
+    array_content = ""
+    inside_quotes = False
+    
+    for char in line:
+        if char == ',' and not inside_array and not inside_quotes:
+            if array_content:
+                elements.append(parse_csv_line(array_content[1:-1]))
+                array_content = ""
+            else:
+                elements.append(current_element.strip())
+            current_element = ""
+        elif char == '[' and not inside_quotes:
+            inside_array = True
+            array_content += char
+        elif char == ']' and not inside_quotes:
+            inside_array = False
+            array_content += char
+        elif char == "'":
+            inside_quotes = not inside_quotes
+            current_element += char
+        else:
+            if inside_array:
+                array_content += char
+            else:
+                current_element += char
+    
+    if current_element:
+        if current_element.startswith("'") and current_element.endswith("'"):
+            current_element = current_element[1:-1]
+        elements.append(current_element.strip())
+    elif array_content:
+        elements.append(parse_csv_line(array_content[1:-1]))
+    
+    return elements
+
+
+def find():
+  for folder in utils.listSubdirs(utils.getRoot("data/scans/")):
+    for file in utils.listSubdirs(utils.getRoot(f"data/scans/{folder}")):
+      with open(utils.getRoot(f"data/scans/{folder}/{file}"), "r") as file:
+        lines = file.readlines()
+        for line in lines:
+          data = parse_csv_line(line)
+          if data[1] != "1":
+            continue
+          print(f'{data[0]} ({data[2]}), {data[4]}')
+    
+
+
+
 
 def getMostCommon(protocol: str, n: int):
   nmap_services_file = "/usr/share/nmap/nmap-services"

modules/main/Dashboard.html

@@ -1,19 +1,112 @@
 
 
 <main class="container">
+  <h2>Auto-Shodanner</h2>
+  <p id="scanText">No scan is running, but this is where statistics will show up!</p>
+  <canvas id="pingChart" style="display: none;"></canvas>
+  <canvas id="threadChart" style="display: none;"></canvas>
 </main>
 
+<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
 <script>
+const scanText = document.getElementById('scanText')
+
+const pingChartctx = document.getElementById('pingChart')
+const pingChart = new Chart(pingChartctx, {
+  type: 'line',
+  responsive: true,
+  labels: Array.from(Array(10).keys()),
+  data: {
+    datasets: [{
+      label: 'IPs UP Per second',
+      data: [],
+      borderWidth: 1
+    },{
+      label: 'IPs DOWN Per second',
+      data: [],
+      borderWidth: 1
+    },{
+      label: 'IPs with ports Per second',
+      data: [],
+      borderWidth: 1
+    }]
+  },
+  options: {
+    scales: {
+      x: {
+        type: 'linear'
+      }
+    }
+  }
+})
+
+const threadChartctx = document.getElementById('threadChart')
+const threadChart = new Chart(threadChartctx, {
+  type: 'line',
+  responsive: true,
+  labels: Array.from(Array(10).keys()),
+  data: {
+    datasets: [{
+      label: 'Ping scanning',
+      fill: true,
+      data: [],
+      borderWidth: 1
+    },{
+      label: 'Nmap scanning',
+      fill: true,
+      data: [],
+      borderWidth: 1
+    },{
+      label: 'Extended scanning',
+      fill: true,
+      data: [],
+      borderWidth: 1
+    }]
+  },
+  options: {
+    scales: {
+      x: {
+        type: 'linear'
+      },
+      y: {
+        min: 0,
+        max: 500
+      }
+    }
+  }
+})
+
+let dataCount = 0
 
 window.main = () => {
   window.addListener('Scanner-Metrics', (data)=>{
-    document.getElementsByClassName("container")[0].innerHTML = `
+    data = data.data
-    <h1>Auto-Shodanner</h1>
+    utils.getel("pingChart").style.display = ""
-    <h3>Addresses Scanned: ${data.data.scanCount} (${data.data.scanCount/42949672.96}% Of the internet.)</h3>
+    utils.getel("threadChart").style.display = ""
-    `
+
+    pingChart.data.datasets[0].data.push(data.upIpsPS)
+    pingChart.data.datasets[1].data.push(data.downIpsPS)
+    pingChart.data.datasets[2].data.push(data.extendedIpsPS)
+
+    pingChart.data.labels.push(dataCount)
+    pingChart.update()
+
+    threadChart.data.datasets[0].data.push(data.hostSearchingCount)
+    threadChart.data.datasets[1].data.push(data.hostSearchingCount+data.nmapScanningCount)
+    threadChart.data.datasets[2].data.push(data.numJobs)
+
+    threadChart.data.labels.push(dataCount)
+    threadChart.update()
+
+    dataCount += 1
+
+    scanText.innerText = '' +
+    `Total addresses scanned: ${data.countScannedBeforeStart+data.upIps+data.downIps}\n` +
+    `Down this session: ${data.downIps}\n` +
+    `Up this session: ${data.upIps}\n` +
+    `With ports this session: ${data.extendedIps}`
   })
 }
 
 
-
 </script>

modules/main/main.py

@@ -204,4 +204,5 @@ def deleteUser(ac, data):
   
   mm.deleteUser(user)
   mm.sendPopupSuccess(ac.rawClient, "Success", "User deleted!")  
-  loadSessionsAdmin(ac)
+  loadSessionsAdmin(ac)
+  

modules/scan/main.py

@@ -45,21 +45,29 @@ def setSettings(ac, data):
           valid = False
       
   if valid:
-    print(data)
+    # print(data)
     mm.vars['Scanner-Settings'] = data
   else:
     mm.sendPopupError(ac.rawClient, "Error", "There is an error in the config.")
 
 
+def onStats(stats):
+  print(stats)
+  for ac in mm.authServer.clients:
+    if ac.currentPage == "/main/dashboard":
+      ac.send("Scanner-Metrics", stats)
+
+
 def startScanner(ac, data):
-  scan.start(mm.vars['Scanner-Settings'])
+  mm.sendPopupSuccess(ac.rawClient, "Scanner", "Scanner Started!")
+  scan.start(mm.vars['Scanner-Settings'], onStats)
   
   
 def stopScanner(ac, data):
+  mm.sendPopupSuccess(ac.rawClient, "Scanner", "Scanner Stopped!")
   scan.stop()
 
 
-
 def init(moduleMaster):
   global mm
   mm = moduleMaster
@@ -97,9 +105,4 @@ def init(moduleMaster):
   mm.addAuthEventListener('Scanner-StopScanner', stopScanner)
 
 def main():
-  while True:
+  pass
-    if scan.processStarted():
-      print("eee")
-      # print(scan.getStdout())
-    # print("eee")
-    time.sleep(1)

test.py

@@ -1,55 +1,2 @@
 import src.utils as utils
 
-def parse_csv_line(line):
-    elements = []
-    current_element = ""
-    inside_array = False
-    array_content = ""
-    inside_quotes = False
-    
-    for char in line:
-        if char == ',' and not inside_array and not inside_quotes:
-            if array_content:
-                elements.append(parse_csv_line(array_content[1:-1]))
-                array_content = ""
-            else:
-                elements.append(current_element.strip())
-            current_element = ""
-        elif char == '[' and not inside_quotes:
-            inside_array = True
-            array_content += char
-        elif char == ']' and not inside_quotes:
-            inside_array = False
-            array_content += char
-        elif char == "'":
-            inside_quotes = not inside_quotes
-            current_element += char
-        else:
-            if inside_array:
-                array_content += char
-            else:
-                current_element += char
-    
-    if current_element:
-        if current_element.startswith("'") and current_element.endswith("'"):
-            current_element = current_element[1:-1]
-        elements.append(current_element.strip())
-    elif array_content:
-        elements.append(parse_csv_line(array_content[1:-1]))
-    
-    return elements
-
-
-def find():
-  # print("started!")
-  for folder in utils.listSubdirs(utils.getRoot("data/scans/")):
-    for file in utils.listSubdirs(utils.getRoot(f"data/scans/{folder}")):
-      with open(utils.getRoot(f"data/scans/{folder}/{file}"), "r") as file:
-        lines = file.readlines()
-        for line in lines:
-          data = parse_csv_line(line)
-          if data[1] != "1":
-            continue
-          print(f'{data[0]} ({data[2]}), {data[4]}')
-    
-  # print("sotopped!!")