Add cmdline tools, database, port scanning, and searching

.gitignore

@@ -12,3 +12,4 @@ Cargo.lock
 
 # MSVC Windows builds of rustc generate these, which store debugging information
 *.pdb
+ping_result_database/

Cargo.toml

@@ -4,7 +4,13 @@ version = "0.1.0"
 edition = "2024"
 
 [dependencies]
-loadingbar = "1.0.1"
+bincode = { version = "2.0.1", features = ["serde"] }
+byteorder = "1.5.0"
+indicatif = "0.17.11"
+memchr = "2.7.4"
 pnet = "0.35.0"
 rand = "0.9.0"
 rocksdb = "0.23.0"
+serde = { version = "1.0.219", features = ["derive"] }
+serde_json = "1.0.140"
+tokio = "1.44.2"

src/database.rs

@@ -0,0 +1,624 @@
+use std::{
+    collections::HashSet,
+    io::Error,
+    net::IpAddr,
+    sync::{Arc, Mutex},
+    time::Instant,
+};
+
+use byteorder::{BigEndian, ByteOrder, WriteBytesExt};
+use memchr::memmem;
+use rocksdb::{Cache, ColumnFamily, DB, IteratorMode, Options, ReadOptions, WriteBatch};
+use serde::{Deserialize, Serialize};
+
+use crate::port_scan::port_scan::ScanResult;
+
+static COLUMN_COUNT: usize = 5;
+static TEST_ROW_COUNT: usize = 1000;
+
+// Global settings for optimal performance
+const BLOCK_CACHE_SIZE_MB: usize = 512; // 512MB block cache
+const WRITE_BUFFER_SIZE_MB: usize = 64; // 64MB write buffer
+const NUM_PARALLEL_THREADS: usize = 8; // Number of threads for parallel operations
+const BATCH_SIZE: usize = 1000; // Batch size for writes
+
+pub struct ResultDatabase {
+    pub path: String,
+    options: Options,
+    columns: Vec<String>,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct StringRow {
+    pub id: String,          // Row identifier
+    pub values: Vec<String>, // Array of string values
+}
+
+impl StringRow {
+    pub fn to_string(&self) -> String {
+        let mut str = "".to_string();
+
+        str += format!("Row ID: {}, Values: [", self.id).as_str();
+        for (i, value) in self.values.iter().enumerate() {
+            if i > 0 {
+                str += ", ";
+            }
+            str += format!("{}: \"{}\"", i, value).as_str();
+        }
+        str += "]";
+
+        str
+    }
+}
+
+/// Enum for defining search criteria
+#[derive(Debug)]
+enum SearchCriteria {
+    ByColumnValue(usize, String), // Search by specific column value
+    ByColumnPrefix(usize, String), // Search by column value prefix
+                                  // ByIdRange(String, String),     // Search by ID range
+}
+
+impl ResultDatabase {
+    pub fn new(path: &str) -> Self {
+        let mut options = Options::default();
+
+        options.create_if_missing(true);
+        options.create_missing_column_families(true);
+        options.increase_parallelism(NUM_PARALLEL_THREADS as i32); // Use multiple background threads
+        options.set_max_background_jobs(4);
+        options.set_write_buffer_size(WRITE_BUFFER_SIZE_MB * 1024 * 1024); // Larger write buffer
+        options.set_max_write_buffer_number(3); // Allow more write buffers
+        options.set_target_file_size_base(64 * 1024 * 1024); // 64MB per SST file
+        options.set_level_zero_file_num_compaction_trigger(4); // Start compaction after 4 L0 files
+        options.set_level_zero_slowdown_writes_trigger(16); // Start slowing down writes after 16 L0 files
+        options.set_level_zero_stop_writes_trigger(24); // Stop writes after 24 L0 files
+        options.set_max_bytes_for_level_base(512 * 1024 * 1024); // 512MB for base level
+        options.set_disable_auto_compactions(false); // Enable auto compactions
+        options.optimize_level_style_compaction(WRITE_BUFFER_SIZE_MB * 1024 * 1024);
+        options.set_max_total_wal_size(256 * 1024 * 1024); // 256MB max for WAL files
+        options.set_keep_log_file_num(5); // Keep 5 log files
+        options.set_log_level(rocksdb::LogLevel::Warn); // Minimal logging
+
+        // Set up block cache for improved read performance
+        let mut block_opts = rocksdb::BlockBasedOptions::default();
+        block_opts.set_block_cache(&Cache::new_lru_cache(BLOCK_CACHE_SIZE_MB * 1024 * 1024));
+        block_opts.set_bloom_filter(10.0, false);
+        block_opts.set_whole_key_filtering(true);
+        block_opts.set_cache_index_and_filter_blocks(true);
+        block_opts.set_pin_l0_filter_and_index_blocks_in_cache(true);
+        options.set_block_based_table_factory(&block_opts);
+
+        // Define column families for different indexes
+
+        let mut column_families = vec!["default".to_string()]; // Main data store
+
+        // Add column families for each column index we might want to search by
+        // (for demo, we'll create indexes for 5 potential columns)
+        for i in 0..COLUMN_COUNT {
+            column_families.push(format!("col{}_idx", i).to_string());
+        }
+
+        Self {
+            path: path.to_string(),
+            options,
+            columns: column_families,
+        }
+    }
+
+    pub fn add_ping_results(
+        &self,
+        results: &Vec<IpAddr>,
+    ) -> Result<(), Box<dyn std::error::Error>> {
+        let mut string_rows = Vec::with_capacity(results.len()); // Pre-allocate capacity
+
+        for result in results {
+            string_rows.push(StringRow {
+                id: result.to_string(),
+                values: vec![],
+            });
+        }
+
+        return self.save_rows(string_rows);
+    }
+
+    pub fn add_tcp_results(
+        &self,
+        results: &Vec<ScanResult>,
+    ) -> Result<(), Box<dyn std::error::Error>> {
+        let mut string_rows = Vec::with_capacity(results.len()); // Pre-allocate capacity
+
+        for result in results {
+            string_rows.push(result.to_string_row());
+        }
+
+        return self.save_rows(string_rows);
+    }
+
+    pub fn save_rows(&self, string_rows: Vec<StringRow>) -> Result<(), Box<dyn std::error::Error>> {
+        let db = Arc::new(DB::open_cf(&self.options, &self.path, &self.columns)?);
+        let cf_default = db.cf_handle("default").unwrap();
+
+        // Get handles to column index families
+        let mut cf_columns = Vec::new();
+        for i in 0..3 {
+            let cf = db.cf_handle(&format!("col{}_idx", i)).unwrap();
+            cf_columns.push(cf);
+        }
+
+        let start = Instant::now();
+        let length = string_rows.len();
+
+        // Split the rows into chunks for parallel processing
+        let chunks: Vec<Vec<StringRow>> = string_rows
+            .chunks(BATCH_SIZE)
+            .map(|chunk| chunk.to_vec())
+            .collect();
+
+        // Process chunks in parallel
+        let elapsed = {
+            let db_ref = Arc::clone(&db);
+            let cf_default_ref = cf_default;
+            let cf_columns_ref = &cf_columns;
+
+            // Create batches in parallel but write them sequentially
+            let batches: Vec<WriteBatch> = chunks
+                .into_iter()
+                .map(|chunk| {
+                    let mut batch = WriteBatch::default();
+
+                    for row in chunk {
+                        // Use optimized binary format for the main data
+                        let mut data = Vec::with_capacity(256);
+
+                        // Format: id_len + id + count + (len + str) for each value
+                        // Binary format: direct encoding without JSON overhead
+                        encode_row_binary(&mut data, &row);
+
+                        // Store in main column family
+                        batch.put_cf(cf_default_ref, row.id.as_bytes(), &data);
+
+                        // Create indexes only for searchable columns (0-2)
+                        for (col_idx, value) in row.values.iter().enumerate() {
+                            if col_idx < cf_columns_ref.len() {
+                                // Create search-friendly keys: value:rowid
+                                // Use minimal escaping for better performance
+                                let idx_key = format!("{}:{}", fast_escape(value), row.id);
+                                batch.put_cf(
+                                    cf_columns_ref[col_idx],
+                                    idx_key.as_bytes(),
+                                    row.id.as_bytes(),
+                                );
+                            }
+                        }
+                    }
+
+                    batch
+                })
+                .collect();
+
+            // Write all batches to the database
+            for batch in batches {
+                db_ref.write(batch)?;
+            }
+
+            // Force a flush to ensure all data is persisted
+            db_ref.flush()?;
+
+            start.elapsed()
+        };
+
+        println!("Saved {} rows in {}ms", length, elapsed.as_millis());
+
+        Ok(())
+    }
+
+    pub fn get_row_by_host(&self, row: &str) -> Option<StringRow> {
+        let db = DB::open_cf(&self.options, &self.path, &self.columns);
+        if db.is_err() {
+            return None;
+        };
+        let db = db.unwrap();
+        let cf_default = db.cf_handle("default").unwrap();
+
+        return fetch_row(&db, cf_default, row);
+    }
+
+    pub fn get_rows_by_port(&self, port: &str) -> Vec<StringRow> {
+        if let Ok(result) = self.search_substring_in_column(self.columns[0].as_str(), port) {
+            return result;
+        } else {
+            return Vec::new();
+        }
+    }
+
+    pub fn search_substring_in_column(
+        &self,
+        column: &str,
+        substring: &str,
+    ) -> Result<Vec<StringRow>, rocksdb::Error> {
+        let db = Arc::new(DB::open_cf(&self.options, &self.path, &self.columns)?);
+
+        let cf = db.cf_handle(column).unwrap();
+
+        let mut matching_keys: Vec<StringRow> = Vec::new();
+
+        // Use RocksDB's iterator for efficient scanning
+        let iter = db.iterator_cf(cf, IteratorMode::Start);
+
+        // Iterate through all key-value pairs in the column family
+        for item in iter {
+            let (key_bytes, value_bytes) = item?;
+
+            // Convert value to string (assumes UTF-8 encoding)
+            if let Ok(value_str) = std::str::from_utf8(&value_bytes) {
+                // Check if the value contains the substring
+                if value_str.contains(substring) {
+                    // Convert key to string and add to results
+                    if let Ok(key_str) = std::str::from_utf8(&key_bytes) {
+                        if let Some(row) = decode_row_binary(&value_bytes) {
+                            matching_keys.push(row);
+                        }
+                    }
+                }
+            }
+        }
+
+        Ok(matching_keys)
+    }
+}
+
+// Count results from a search without printing
+fn search(
+    db: &DB,
+    cf_default: &ColumnFamily,
+    cf_columns: &[&ColumnFamily],
+    criteria: SearchCriteria,
+) -> Result<Vec<StringRow>, Box<dyn std::error::Error>> {
+    let mut results: Vec<StringRow> = Vec::new();
+
+    match criteria {
+        SearchCriteria::ByColumnValue(col_idx, value) => {
+            if col_idx >= cf_columns.len() {
+                return Ok(results);
+            }
+
+            // Create search key with escaped value
+            let prefix = format!("{}:", fast_escape(&value));
+            let mut opts = ReadOptions::default();
+            opts.set_prefix_same_as_start(true);
+
+            let iterator = db.iterator_cf_opt(
+                cf_columns[col_idx],
+                opts,
+                rocksdb::IteratorMode::From(prefix.as_bytes(), rocksdb::Direction::Forward),
+            );
+
+            for item in iterator {
+                let (idx_key, data) = item?;
+                let idx_key_str = String::from_utf8(idx_key.to_vec())?;
+
+                // Skip if we've moved past our prefix
+                if !idx_key_str.starts_with(&prefix) {
+                    break;
+                }
+
+                let row = decode_row_binary(&data);
+
+                if let Some(row) = row {
+                    results.push(row);
+                }
+            }
+        }
+
+        SearchCriteria::ByColumnPrefix(col_idx, prefix) => {
+            if col_idx >= cf_columns.len() {
+                return Ok(results);
+            }
+
+            // Create search key with escaped prefix
+            let search_prefix = fast_escape(&prefix);
+
+            let iterator = db.iterator_cf(
+                cf_columns[col_idx],
+                rocksdb::IteratorMode::From(search_prefix.as_bytes(), rocksdb::Direction::Forward),
+            );
+
+            for item in iterator {
+                let (idx_key, data) = item?;
+                let idx_key_str = String::from_utf8(idx_key.to_vec())?;
+
+                // Extract just the value part of the index key
+                let parts: Vec<&str> = idx_key_str.splitn(2, ':').collect();
+                if parts.len() < 2 {
+                    continue;
+                }
+
+                let value_part = fast_unescape(parts[0]);
+
+                // Skip if value doesn't start with our prefix
+                if !value_part.starts_with(&prefix) {
+                    // If we've moved past potential matches, break early
+                    if value_part > prefix {
+                        break;
+                    }
+                    continue;
+                }
+
+                let row = decode_row_binary(&data);
+
+                if let Some(row) = row {
+                    results.push(row);
+                }
+            }
+        }
+    }
+
+    Ok(results)
+}
+
+// Fast minimal escaping for key values
+#[inline]
+fn fast_escape(s: &str) -> String {
+    // Only escape the colon character which is our separator
+    s.replace(":", "\\:")
+}
+
+// Fast unescaping for key values
+#[inline]
+fn fast_unescape(s: &str) -> String {
+    // Only unescape the colon
+    s.replace("\\:", ":")
+}
+
+// Fast direct row fetch by ID
+fn fetch_row(db: &DB, cf_default: &ColumnFamily, row_id: &str) -> Option<StringRow> {
+    match db.get_cf(cf_default, row_id.as_bytes()) {
+        Ok(Some(value)) => decode_row_binary(&value),
+        _ => None,
+    }
+}
+
+// Fast column value fetch
+fn fetch_column(db: &DB, cf_default: &ColumnFamily, row_id: &str, column_idx: usize) -> String {
+    match fetch_row(db, cf_default, row_id) {
+        Some(row) => get_column_value(&row, column_idx),
+        None => String::new(),
+    }
+}
+
+// Get a column value, returning empty string if column doesn't exist
+#[inline]
+fn get_column_value(row: &StringRow, column_index: usize) -> String {
+    if column_index < row.values.len() {
+        row.values[column_index].clone()
+    } else {
+        String::new() // Return empty string for missing columns
+    }
+}
+
+// Binary decoding of row data
+fn decode_row_binary(data: &[u8]) -> Option<StringRow> {
+    if data.len() < 8 {
+        return None;
+    }
+
+    let mut pos = 0;
+
+    // Read ID length
+    let mut id_len_bytes = [0u8; 4];
+    id_len_bytes.copy_from_slice(&data[pos..pos + 4]);
+    let id_len = u32::from_le_bytes(id_len_bytes) as usize;
+    pos += 4;
+
+    // Read ID
+    if pos + id_len > data.len() {
+        return None;
+    }
+    let id = String::from_utf8_lossy(&data[pos..pos + id_len]).to_string();
+    pos += id_len;
+
+    // Read number of values
+    if pos + 4 > data.len() {
+        return None;
+    }
+    let mut values_count_bytes = [0u8; 4];
+    values_count_bytes.copy_from_slice(&data[pos..pos + 4]);
+    let values_count = u32::from_le_bytes(values_count_bytes) as usize;
+    pos += 4;
+
+    // Read values
+    let mut values = Vec::with_capacity(values_count);
+    for _ in 0..values_count {
+        if pos + 4 > data.len() {
+            return None;
+        }
+
+        let mut value_len_bytes = [0u8; 4];
+        value_len_bytes.copy_from_slice(&data[pos..pos + 4]);
+        let value_len = u32::from_le_bytes(value_len_bytes) as usize;
+        pos += 4;
+
+        if pos + value_len > data.len() {
+            return None;
+        }
+
+        let value = String::from_utf8_lossy(&data[pos..pos + value_len]).to_string();
+        values.push(value);
+        pos += value_len;
+    }
+
+    Some(StringRow { id, values })
+}
+
+// Binary encoding of row data for maximum performance
+fn encode_row_binary(buf: &mut Vec<u8>, row: &StringRow) {
+    // Write ID length and ID
+    let id_bytes = row.id.as_bytes();
+    buf.extend_from_slice(&(id_bytes.len() as u32).to_le_bytes());
+    buf.extend_from_slice(id_bytes);
+
+    // Write number of values
+    buf.extend_from_slice(&(row.values.len() as u32).to_le_bytes());
+
+    // Write each value
+    for value in &row.values {
+        let value_bytes = value.as_bytes();
+        buf.extend_from_slice(&(value_bytes.len() as u32).to_le_bytes());
+        buf.extend_from_slice(value_bytes);
+    }
+}
+
+// fn benchmark_create_rows() {
+//     for i in 0..10000 {
+//         // Generate 10,000 test rows
+//         let mut values = Vec::with_capacity(5);
+
+//         // Add IP address (column 0)
+//         if i % 3 == 0 {
+//             values.push(format!("192.168.1.{}", i % 255));
+//         } else if i % 3 == 1 {
+//             values.push(format!("10.0.{}.{}", (i / 255) % 255, i % 255));
+//         } else {
+//             values.push(format!("172.16.{}.{}", (i / 255) % 255, i % 255));
+//         }
+
+//         // Add status (column 1)
+//         if i % 5 < 4 {
+//             // 80% active
+//             values.push("active".to_string());
+//         } else {
+//             values.push("inactive".to_string());
+//         }
+
+//         // Add response time (column 2) for active servers
+//         if i % 5 < 4 {
+//             values.push(format!("{}ms", (i % 100) + 1));
+//         }
+
+//         // Add server name (column 3)
+//         if i % 2 == 0 {
+//             values.push(format!("server{:04}", i));
+//         }
+
+//         // Add priority (column 4) for some servers
+//         if i % 7 == 0 {
+//             values.push("high_priority".to_string());
+//         } else if i % 11 == 0 {
+//             values.push("low_priority".to_string());
+//         }
+
+//         // string_rows.push(StringRow {
+//         //     id: format!("row{:06}", i),
+//         //     values,
+//         // });
+//     }
+// }
+
+// // Benchmark search performance
+// fn benchmark_search<F>(
+//     db: &DB,
+//     cf_default: &ColumnFamily,
+//     cf_columns: &[&ColumnFamily],
+//     name: &str,
+//     criteria_fn: F,
+// ) -> Result<(), Box<dyn std::error::Error>>
+// where
+//     F: Fn() -> SearchCriteria,
+// {
+//     let mut total_duration = Duration::from_secs(0);
+//     let mut total_results = 0;
+
+//     for i in 1..=3 {
+//         let criteria = criteria_fn();
+//         let start = Instant::now();
+//         let count = count_search_results(db, cf_default, cf_columns, criteria)?;
+//         let duration = start.elapsed();
+
+//         total_duration += duration;
+//         total_results = count; // All runs should return same count
+
+//         println!("  Run {}: Found {} results in {:?}", i, count, duration);
+//     }
+
+//     let avg_duration = total_duration / 3;
+//     println!(
+//         "  Average: {:?} for {} results",
+//         avg_duration, total_results
+//     );
+//     println!(
+//         "  Speed: {:.2} results/ms",
+//         total_results as f64 / avg_duration.as_millis() as f64
+//     );
+
+//     Ok(())
+// }
+
+// // Benchmark direct row fetch performance
+// fn benchmark_direct_fetch(
+//     db: &DB,
+//     cf_default: &ColumnFamily,
+//     name: &str,
+//     row_id: &str,
+// ) -> Result<(), Box<dyn std::error::Error>> {
+//     let mut total_duration = Duration::from_secs(0);
+
+//     for i in 1..=3 {
+//         let start = Instant::now();
+
+//         // Do multiple fetches to get a measurable time
+//         for _ in 0..1000 {
+//             let _ = fetch_row(db, cf_default, row_id);
+//         }
+
+//         let duration = start.elapsed();
+//         total_duration += duration;
+
+//         println!("  Run {}: 1000 row fetches in {:?}", i, duration);
+//     }
+
+//     let avg_duration = total_duration / 3;
+//     println!("  Average: {:?} for 1000 fetches", avg_duration);
+//     println!(
+//         "  Speed: {:.2} fetches/ms",
+//         1000.0 / avg_duration.as_millis() as f64
+//     );
+
+//     Ok(())
+// }
+
+// // Benchmark column fetch performance
+// fn benchmark_column_fetch(
+//     db: &DB,
+//     cf_default: &ColumnFamily,
+//     name: &str,
+//     row_id: &str,
+//     col_idx: usize,
+// ) -> Result<(), Box<dyn std::error::Error>> {
+//     let mut total_duration = Duration::from_secs(0);
+
+//     for i in 1..=3 {
+//         let start = Instant::now();
+
+//         // Do multiple fetches to get a measurable time
+//         for _ in 0..1000 {
+//             let _ = fetch_column(db, cf_default, row_id, col_idx);
+//         }
+
+//         let duration = start.elapsed();
+//         total_duration += duration;
+
+//         println!("  Run {}: 1000 column fetches in {:?}", i, duration);
+//     }
+
+//     let avg_duration = total_duration / 3;
+//     println!("  Average: {:?} for 1000 fetches", avg_duration);
+//     println!(
+//         "  Speed: {:.2} fetches/ms",
+//         1000.0 / avg_duration.as_millis() as f64
+//     );
+
+//     Ok(())
+// }
+//
+// // Example usage with batching for very large datasets

src/lib.rs

@@ -1,2 +1,4 @@
+pub mod database;
 pub mod online_scan;
 pub mod parse_ip_range;
+pub mod port_scan;

src/main.rs

@@ -1,28 +1,207 @@
-pub mod online_scan;
+use std::{env, net::IpAddr, time::Duration};
-pub mod parse_ip_range;
 
-use std::env;
+use online_scan::PingResult;
-
-use online_scan::ping_scanner;
 use parse_ip_range::parse_ip_targets;
+use untitled::{database::ResultDatabase, online_scan, parse_ip_range, port_scan::tcp_scan};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let args: Vec<String> = env::args().collect();
+    let database = ResultDatabase::new("ping_result_database");
 
+    if args.len() <= 1 {
+        println!("You must specify a command!");
+        print_help(None);
+    }
+
+    match args[1].to_lowercase().as_str() {
+        "scan" => {
+            if args.len() != 4 {
+                println!("Invalid Usage!");
+                print_help(Some(args[1].as_str()));
+                return Ok(());
+            }
+            let _ = scan(database, args[2].clone(), args[3].clone());
+        }
+        "search" => {
+            if args.len() != 4 {
+                println!("Invalid Usage!");
+                print_help(Some(args[1].as_str()));
+                return Ok(());
+            }
+            search(database, args[2].to_string(), args[3].to_string());
+        }
+        "help" => {
+            if args.len() != 3 {
+                print_help(None);
+                return Ok(());
+            }
+            print_help(Some(args[2].as_str()));
+        }
+        _ => {
+            println!("Invalid command!");
+            print_help(None);
+            return Ok(());
+        }
+    }
+
+    Ok(())
+}
+fn scan(
+    database: ResultDatabase,
+    search_type: String,
+    arg: String,
+) -> Result<(), Box<dyn std::error::Error>> {
     // Set default targets or use command line input
-    let targets = if args.len() > 1 {
+    let targets = arg;
-        args[1].clone()
-    } else {
-        "".to_string()
-    };
 
     // Parse the targets into IP addresses
     let hosts = parse_ip_targets(&targets)?;
 
-    let length = hosts.len();
+    match search_type.as_str() {
+        "ping" => {
+            let length = hosts.len();
 
-    let results = ping_scanner::ping_scan(hosts).unwrap();
+            let up_hosts: Vec<IpAddr> = online_scan::ping_scanner::ping_scan(hosts).unwrap();
+            println!("Finished! {} Scanned, {} Up", length, up_hosts.len());
+            let _ = database.add_ping_results(&up_hosts);
+        }
+        "tcp" => {
+            let length = hosts.len();
+
+            let up_hosts: Vec<IpAddr> = online_scan::ping_scanner::ping_scan(hosts).unwrap();
+            println!("Finished! {} Scanned, {} Up", length, up_hosts.len());
+            let _ = database.add_ping_results(&up_hosts);
+
+            let tcp_ports = vec![
+                1, 3, 4, 6, 7, 9, 13, 17, 19, 20, 21, 22, 23, 24, 25, 26, 30, 32, 33, 37, 42, 43,
+                49, 53, 70, 79, 80, 81, 82, 83, 84, 85, 88, 89, 90, 99, 100, 106, 109, 110, 111,
+                113, 119, 125, 135, 139, 143, 144, 146, 161, 163, 179, 199, 211, 212, 222, 254,
+                255, 256, 259, 264, 280, 301, 306, 311, 340, 366, 389, 406, 407, 416, 417, 425,
+                427, 443, 444, 445, 458, 464, 465, 481, 497, 500, 512, 513, 514, 515, 524, 541,
+                543, 544, 545, 548, 554, 555, 563, 587, 593, 616, 617, 625, 631, 636, 646, 648,
+                666, 667, 668, 683, 687, 691, 700, 705, 711, 714, 720, 722, 726, 749, 765, 777,
+                783, 787, 800, 801, 808, 843, 873, 880, 888, 898, 900, 901, 902, 903, 911, 912,
+                981, 987, 990, 992, 993, 995, 999, 1000, 1001, 1002, 1007, 1009, 1010, 1011, 1021,
+                1022, 1023, 1024, 1025, 1026, 1027, 1028, 1029, 1030, 1031, 1032, 1033, 1034, 1035,
+                1036, 1037, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047, 1048, 1049,
+                1050, 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, 1059, 1060, 1061, 1062, 1063,
+                1064, 1065, 1066, 1067, 1068, 1069, 1070, 1071, 1072, 1073, 1074, 1075, 1076, 1077,
+                1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, 1088, 1089, 1090, 1091,
+                1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1100, 1102, 1104, 1105, 1106, 1107,
+                1108, 1110, 1111, 1112, 1113, 1114, 1117, 1119, 1121, 1122, 1123, 1124, 1126, 1130,
+                1131, 1132, 1137, 1138, 1141, 1145, 1147, 1148, 1149, 1151, 1152, 1154, 1163, 1164,
+                1165, 1166, 1169, 1174, 1175, 1183, 1185, 1186, 1187, 1192, 1198, 1199, 1201, 1213,
+                1216, 1217, 1218, 1233, 1234, 1236, 1244, 1247, 1248, 1259, 1271, 1272, 1277, 1287,
+                1296, 1300, 1301, 1309, 1310, 1311, 1322, 1328, 1334, 1352, 1417, 1433, 1434, 1443,
+                1455, 1461, 1494, 1500, 1501, 1503, 1521, 1524, 1533, 1556, 1580, 1583, 1594, 1600,
+                1641, 1658, 1666, 1687, 1688, 1700, 1717, 1718, 1719, 1720, 1721, 1723, 1755, 1761,
+                1782, 1783, 1801, 1805, 1812, 1839, 1840, 1862, 1863, 1864, 1875, 1900, 1914, 1935,
+                1947, 1971, 1972, 1974, 1984, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+                2007, 2008, 2009, 2010, 2013, 2020, 2021, 2022, 2030, 2033, 2034, 2035, 2038, 2040,
+                2041, 2042, 2043, 2045, 2046, 2047, 2048, 2049, 2065, 2068, 2099, 2100, 2103, 2105,
+                2106, 2107, 2111, 2119, 2121, 2126, 2135, 2144, 2160, 2161, 2170, 2179, 2190, 2191,
+                2196, 2200, 2222, 2251, 2260, 2288, 2301, 2323, 2366, 2381, 2382, 2383, 2393, 2394,
+                2399, 2401, 2492, 2500, 2522, 2525, 2557, 2601, 2602, 2604, 2605, 2607, 2608, 2638,
+                2701, 2702, 2710, 2717, 2718, 2725, 2800, 2809, 2811, 2869, 2875, 2909, 2910, 2920,
+                2967, 2968, 2998, 3000, 3001, 3003, 3005, 3006, 3007, 3011, 3013, 3017, 3030, 3031,
+                3052, 3071, 3077, 3128, 3168, 3211, 3221, 3260, 3261, 3268, 3269, 3283, 3300, 3301,
+                3306, 3322, 3323, 3324, 3325, 3333, 3351, 3367, 3369, 3370, 3371, 3372, 3389, 3390,
+                3404, 3476, 3493, 3517, 3527, 3546, 3551, 3580, 3659, 3689, 3690, 3703, 3737, 3766,
+                3784, 3800, 3801, 3809, 3814, 3826, 3827, 3828, 3851, 3869, 3871, 3878, 3880, 3889,
+                3905, 3914, 3918, 3920, 3945, 3971, 3986, 3995, 3998, 4000, 4001, 4002, 4003, 4004,
+                4005, 4006, 4045, 4111, 4125, 4126, 4129, 4224, 4242, 4279, 4321, 4343, 4443, 4444,
+                4445, 4446, 4449, 4550, 4567, 4662, 4848, 4899, 4900, 4998, 5000, 5001, 5002, 5003,
+                5004, 5009, 5030, 5033, 5050, 5051, 5054, 5060, 5061, 5080, 5087, 5100, 5101, 5102,
+                5120, 5190, 5200, 5214, 5221, 5222, 5225, 5226, 5269, 5280, 5298, 5357, 5405, 5414,
+                5431, 5432, 5440, 5500, 5510, 5544, 5550, 5555, 5560, 5566, 5631, 5633, 5666, 5678,
+                5679, 5718, 5730, 5800, 5801, 5802, 5810, 5811, 5815, 5822, 5825, 5850, 5859, 5862,
+                5877, 5900, 5901, 5902, 5903, 5904, 5906, 5907, 5910, 5911, 5915, 5922, 5925, 5950,
+                5952, 5959, 5960, 5961, 5962, 5963, 5987, 5988, 5989, 5998, 5999, 6000, 6001, 6002,
+                6003, 6004, 6005, 6006, 6007, 6009, 6025, 6059, 6100, 6101, 6106, 6112, 6123, 6129,
+                6156, 6346, 6389, 6502, 6510, 6543, 6547, 6565, 6566, 6567, 6580, 6646, 6666, 6667,
+                6668, 6669, 6689, 6692, 6699, 6779, 6788, 6789, 6792, 6839, 6881, 6901, 6969, 7000,
+                7001, 7002, 7004, 7007, 7019, 7025, 7070, 7100, 7103, 7106, 7200, 7201, 7402, 7435,
+                7443, 7496, 7512, 7625, 7627, 7676, 7741, 7777, 7778, 7800, 7911, 7920, 7921, 7937,
+                7938, 7999, 8000, 8001, 8002, 8007, 8008, 8009, 8010, 8011, 8021, 8022, 8031, 8042,
+                8045, 8080, 8081, 8082, 8083, 8084, 8085, 8086, 8087, 8088, 8089, 8090, 8093, 8099,
+                8100, 8180, 8181, 8192, 8193, 8194, 8200, 8222, 8254, 8290, 8291, 8292, 8300, 8333,
+                8383, 8400, 8402, 8443, 8500, 8600, 8649, 8651, 8652, 8654, 8701, 8800, 8873, 8888,
+                8899, 8994, 9000, 9001, 9002, 9003, 9009, 9010, 9011, 9040, 9050, 9071, 9080, 9081,
+                9090, 9091, 9099, 9100, 9101, 9102, 9103, 9110, 9111, 9200, 9207, 9220, 9290, 9415,
+                9418, 9485, 9500, 9502, 9503, 9535, 9575, 9593, 9594, 9595, 9618, 9666, 9876, 9877,
+                9878, 9898, 9900, 9917, 9929, 9943, 9944, 9968, 9998, 9999, 10000, 10001, 10002,
+                10003, 10004, 10009, 10010, 10012, 10024, 10025, 10082, 10180, 10215, 10243, 10566,
+                10616, 10617, 10621, 10626, 10628, 10629, 10778, 11110, 11111, 11967, 12000, 12174,
+                12265, 12345, 13456, 13722, 13782, 13783, 14000, 14238, 14441, 14442, 15000, 15002,
+                15003, 15004, 15660, 15742, 16000, 16001, 16012, 16016, 16018, 16080, 16113, 16992,
+                16993, 17877, 17988, 18040, 18101, 18988, 19101, 19283, 19315, 19350, 19780, 19801,
+                19842, 20000, 20005, 20031, 20221, 20222, 20828, 21571, 22939, 23502, 24444, 24800,
+                25734, 25735, 26214, 27000, 27352, 27353, 27355, 27356, 27715, 28201, 30000, 30718,
+                30951, 31038, 31337, 32768, 32769, 32770, 32771, 32772, 32773, 32774, 32775, 32776,
+                32777, 32778, 32779, 32780, 32781, 32782, 32783, 32784, 32785, 33354, 33899, 34571,
+                34572, 34573, 35500, 38292, 40193, 40911, 41511, 42510, 44176, 44442, 44443, 44501,
+                45100, 48080, 49152, 49153, 49154, 49155, 49156, 49157, 49158, 49159, 49160, 49161,
+                49163, 49165, 49167, 49175, 49176, 49400, 49999, 50000, 50001, 50002, 50003, 50006,
+                50300, 50389, 50500, 50636, 50800, 51103, 51493, 52673, 52822, 52848, 52869, 54045,
+                54328, 55055, 55056, 55555, 55600, 56737, 56738, 57294, 57797, 58080, 60020, 60443,
+                61532, 61900, 62078, 63331, 64623, 64680, 65000, 65129, 65389,
+            ];
+
+            let tcp_results = tcp_scan::tcp_scan(up_hosts, tcp_ports, Duration::from_secs(3));
+            let _ = database.add_tcp_results(&tcp_results);
+        }
+        _ => {
+            println!("Invalid search type!");
+        }
+    }
 
-    println!("Finished! {} Scanned, {} Up", length, results.len());
     Ok(())
 }
+
+fn search(database: ResultDatabase, search_type: String, arg: String) {
+    match search_type.as_str() {
+        "host" => {
+            let row = database.get_row_by_host(&arg);
+            if let Some(row) = row {
+                println!("{}", row.to_string());
+            } else {
+                println!("Could not find host by argument {}", arg.as_str());
+            }
+        }
+
+        "port" => {
+            let rows = database.get_rows_by_port(&arg);
+
+            for row in rows {
+                println!("{}", row.to_string());
+            }
+        }
+        _ => {
+            println!("Invalid search type!");
+        }
+    }
+}
+
+fn print_help(arg: Option<&str>) {
+    println!(
+        "{}",
+        match arg {
+            None => {
+                "rust-scan help menu
+Commands:
+    scan   <type> <argument> - scan a block of addresses and check for online using icmp echo
+    search <type> <argument> - Search database
+    help   (command)         - Print help"
+            }
+            Some("pingscan") => {
+                "pingscan <addresses>
+scan a block of addresses and check for online using icmp echo
+Usage: pingscan 10.42.0.1,12.34.0.0-12.34.56.78,127.0.0.0/8"
+            }
+            Some(_) => {
+                print_help(None);
+                ""
+            }
+        }
+    );
+}

src/online_scan/online_scan.rs

@@ -1,7 +1,11 @@
 use std::{net::IpAddr, time::Duration};
 
+use serde::{Deserialize, Serialize};
+
+use crate::database::StringRow;
+
 // Structure to hold ping results
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct PingResult {
     pub host: IpAddr,
     pub is_up: bool,
@@ -16,4 +20,22 @@ impl PingResult {
             response_time: None,
         }
     }
+
+    pub fn to_string_row(&self) -> StringRow {
+        StringRow {
+            id: self.host.to_string(),
+            values: vec![
+                if self.is_up {
+                    "up".to_string()
+                } else {
+                    "down".to_string()
+                },
+                if self.response_time.is_some() {
+                    self.response_time.unwrap().as_millis().to_string()
+                } else {
+                    "None".to_string()
+                },
+            ],
+        }
+    }
 }

src/online_scan/ping_scanner.rs

@@ -1,5 +1,5 @@
+use indicatif::ProgressBar;
 use pnet::packet::ip::IpNextHeaderProtocols;
-use pnet::packet::ipv4::Ipv4OptionNumbers::TR;
 use pnet::packet::{
     Packet,
     icmp::{IcmpTypes, echo_request::MutableEchoRequestPacket},
@@ -8,7 +8,6 @@ use pnet::transport::{
     TransportChannelType, TransportProtocol, icmp_packet_iter, transport_channel,
 };
 use pnet::util::checksum;
-use std::cmp::max;
 use std::collections::HashMap;
 use std::net::IpAddr;
 use std::sync::atomic::{AtomicBool, Ordering};
@@ -18,12 +17,12 @@ use std::time::{Duration, Instant};
 
 static TIMEOUT: Duration = Duration::from_secs(3);
 // static MAX_PINGS_PER_SECOND: u64 = 10000;
-static SEND_DELAY_NANOS: Duration = Duration::from_nanos(500);
+static SEND_DELAY_NANOS: Duration = Duration::from_micros(10);
 
 use crate::online_scan::PingResult;
 
-pub fn ping_scan(hosts: Vec<IpAddr>) -> Result<Vec<PingResult>, Box<dyn std::error::Error>> {
+pub fn ping_scan(hosts: Vec<IpAddr>) -> Result<Vec<IpAddr>, Box<dyn std::error::Error>> {
-    let results = Arc::new(Mutex::new(Vec::<PingResult>::new()));
+    let results = Arc::new(Mutex::new(Vec::<IpAddr>::new()));
 
     // Create a receiver channel for ICMP packets
     let (_, mut rx) = transport_channel(
@@ -45,17 +44,24 @@ pub fn ping_scan(hosts: Vec<IpAddr>) -> Result<Vec<PingResult>, Box<dyn std::err
         let mut iter = icmp_packet_iter(&mut rx);
         let start_time = Instant::now();
         let mut finish_sending_time: Option<Instant> = None;
+        // let mut pb: Option<ProgressBar> = None;
 
         // Keep receiving until timeout or all hosts are accounted for
         loop {
             // Stop reciving loop if timeout is reached
             // let time = finished_sending_time;
-            if finish_sending_time.is_some() && finish_sending_time.unwrap().elapsed() >= TIMEOUT {
+            if finish_sending_time.is_some() {
-                break;
+                let delay = finish_sending_time.unwrap().elapsed();
+                // pb.as_ref().unwrap().set_position(delay.as_millis() as u64);
+                if delay >= TIMEOUT {
+                    // pb.unwrap().finish_and_clear();
+                    break;
+                }
             } else if finish_sending_time.is_none()
                 && recv_finished_sending_time.load(Ordering::Relaxed)
             {
                 finish_sending_time = Some(Instant::now());
+                // pb = Some(ProgressBar::new(TIMEOUT.as_millis() as u64));
                 println!("Waiting {} seconds for timeout...", TIMEOUT.as_secs())
             }
             // if time.is_some() {
@@ -65,7 +71,7 @@ pub fn ping_scan(hosts: Vec<IpAddr>) -> Result<Vec<PingResult>, Box<dyn std::err
             //     break;
             // };
 
-            match iter.next_with_timeout(Duration::from_millis(1)) {
+            match iter.next_with_timeout(Duration::from_millis(3)) {
                 Ok(Some((packet, _))) => {
                     if packet.get_icmp_type() == IcmpTypes::EchoReply {
                         let payload = packet.payload();
@@ -79,11 +85,12 @@ pub fn ping_scan(hosts: Vec<IpAddr>) -> Result<Vec<PingResult>, Box<dyn std::err
                         if let Some(host) = host_option {
                             let mut results = recv_results.lock().unwrap();
                             let response_time = start_time.elapsed();
-                            results.push(PingResult {
+                            results.push(host);
-                                host,
+                            // results.push(PingResult {
-                                is_up: true,
+                            //     host,
-                                response_time: Some(response_time),
+                            //     is_up: true,
-                            });
+                            //     response_time: Some(response_time),
+                            // });
 
                             // println!("Up! {0} {1}ms", host, response_time.as_millis());
                         }
@@ -98,9 +105,10 @@ pub fn ping_scan(hosts: Vec<IpAddr>) -> Result<Vec<PingResult>, Box<dyn std::err
     // Spawn sender threads
 
     let sender_requests = Arc::clone(&requests);
-    let sender_results = Arc::clone(&results);
+    // let sender_results = Arc::clone(&results);
     let sender_finished_sending_time = Arc::clone(&finished_sending_time);
     let sender_handle = thread::spawn(move || {
+        let pb = ProgressBar::new(hosts.len() as u64);
         // let mut last_send_time = Instant::now();
         for (i, host) in hosts.iter().enumerate() {
             let host_clone = host.clone();
@@ -114,26 +122,17 @@ pub fn ping_scan(hosts: Vec<IpAddr>) -> Result<Vec<PingResult>, Box<dyn std::err
                 ids.insert(identifier, host_clone);
             }
 
-            let response = send_ping(host_clone, identifier);
+            let _ = send_ping(host_clone, identifier);
-
-            match response {
-                Ok(_) => {}
-                Err(_) => {
-                    let mut results = sender_results.lock().unwrap();
-                    results.push(PingResult {
-                        host: host_clone,
-                        is_up: false,
-                        response_time: None,
-                    });
-                }
-            }
 
             // let now = Instant::now();
             // let delay = MAX_RATE_NANOS - last_send_time.duration_since(now).as_nanos() as u64;
             // last_send_time = now;
+            if (i % 16) == 0 {
+                pb.inc(16);
+            }
             thread::sleep(SEND_DELAY_NANOS);
         }
-        println!("Finished Sending!");
+        pb.finish_and_clear();
 
         sender_finished_sending_time.swap(true, Ordering::Relaxed);
     });

src/parse_ip_range.rs

@@ -3,6 +3,8 @@ use std::{
     str::FromStr,
 };
 
+use rand::{rng, seq::SliceRandom};
+
 // static MAX_HOSTS: u32 = 1024;
 
 /// Parse a comma-separated list of IP targets
@@ -30,6 +32,8 @@ pub fn parse_ip_targets(targets: &str) -> Result<Vec<IpAddr>, Box<dyn std::error
         }
     }
 
+    ips.shuffle(&mut rng());
+
     Ok(ips)
 }
 

src/port_scan/mod.rs

@@ -1 +1,2 @@
+pub mod port_scan;
 pub mod tcp_scan;

src/port_scan/port_scan.rs

@@ -0,0 +1,37 @@
+use std::{collections::HashMap, net::IpAddr};
+
+use crate::database::StringRow;
+
+#[derive(Debug, Clone)]
+pub struct ScanResult {
+    pub ip: IpAddr,
+    pub open_ports: Vec<i32>,
+    // pub data: HashMap<i32, Vec<u8>>,
+}
+
+impl ScanResult {
+    pub fn new(ip: IpAddr) -> Self {
+        ScanResult {
+            ip,
+            open_ports: Vec::new(),
+            // data: HashMap::new(),
+        }
+    }
+    pub fn to_string_row(&self) -> StringRow {
+        StringRow {
+            id: self.ip.to_string(),
+            values: vec![join_nums(&self.open_ports, ",")],
+        }
+    }
+}
+
+fn join_nums(nums: &Vec<i32>, sep: &str) -> String {
+    // 1. Convert numbers to strings
+    let str_nums: Vec<String> = nums
+        .iter()
+        .map(|n| n.to_string()) // map every integer to a string
+        .collect(); // collect the strings into the vector
+
+    // 2. Join the strings. There's already a function for this.
+    str_nums.join(sep)
+}

src/port_scan/tcp_scan.rs

@@ -1 +1,137 @@
+use std::collections::HashMap;
+use std::net::{IpAddr, Ipv4Addr};
+use std::str::FromStr;
+use std::sync::{Arc, Mutex};
+use std::thread;
+use std::time::Duration;
 
+use indicatif::ProgressBar;
+use pnet::datalink::{self, NetworkInterface};
+use pnet::packet::ip::IpNextHeaderProtocols;
+use pnet::packet::tcp::{MutableTcpPacket, TcpFlags, TcpOption, TcpPacket};
+use pnet::packet::{Packet, tcp};
+use pnet::transport::{self, TransportChannelType};
+use pnet::util::checksum;
+use rand::{random_range, random_ratio};
+
+use super::port_scan::ScanResult;
+
+fn std_to_pnet_ipv4(previous: &IpAddr) -> Ipv4Addr {
+    Ipv4Addr::from_str(previous.to_string().as_str()).unwrap()
+}
+
+// Main scanning function
+pub fn tcp_scan(targets: Vec<IpAddr>, ports: Vec<i32>, timeout: Duration) -> Vec<ScanResult> {
+    // Find network interface
+    let interface = datalink::interfaces()
+        .into_iter()
+        .find(|iface| iface.is_up() && !iface.is_loopback() && !iface.ips.is_empty())
+        .expect("No valid network interface found");
+
+    // Create transport channel for sending and receiving
+    let (mut tx, mut rx) = transport::transport_channel(
+        65535,
+        TransportChannelType::Layer4(pnet::transport::TransportProtocol::Ipv4(
+            IpNextHeaderProtocols::Tcp,
+        )),
+    )
+    .expect("Failed to create transport channel");
+
+    // Shared results
+    let results = Arc::new(Mutex::new(HashMap::<IpAddr, Vec<i32>>::new()));
+
+    // Initialize results map
+    {
+        let mut results_map = results.lock().unwrap();
+        for ip in &targets {
+            results_map.insert(*ip, Vec::new());
+        }
+    }
+
+    let receiver_results = Arc::clone(&results);
+    let receiver_handle = thread::spawn(move || {
+        let start_time = std::time::Instant::now();
+
+        while start_time.elapsed() < timeout {
+            let mut iter = transport::tcp_packet_iter(&mut rx);
+
+            match iter.next() {
+                Ok((packet, addr)) => {
+                    if let Some(tcp) = TcpPacket::new(packet.packet()) {
+                        // Check for SYN+ACK flags (indicating open port)
+                        if tcp.get_flags() == TcpFlags::SYN | TcpFlags::ACK {
+                            let mut results_map = receiver_results.lock().unwrap();
+                            if let Some(open_ports) = results_map.get_mut(&addr) {
+                                open_ports.push(tcp.get_source() as i32);
+                            }
+                        }
+                    }
+                }
+                Err(_) => {
+                    // Just continue on errors
+                    thread::sleep(Duration::from_millis(1));
+                }
+            }
+        }
+    });
+
+    let pb = ProgressBar::new((targets.len() * ports.len()) as u64);
+
+    let source_ip = interface
+        .ips
+        .iter()
+        .find(|ip| ip.is_ipv4())
+        .expect("No IPv4 address found")
+        .ip();
+
+    for target in &targets {
+        for port in &ports {
+            // let source_ip = Ipv4Addr::from_bits(random_range(0..=(0xffffffff)));
+            let source_port: u16 = random_range(1..=65535);
+            // println!("{}", source_ip.to_string());
+
+            let mut tcp_buffer = vec![0u8; 20 + 20]; // IP header + TCP header
+            let mut tcp_header = MutableTcpPacket::new(&mut tcp_buffer[0..]).unwrap();
+
+            tcp_header.set_source(source_port);
+            tcp_header.set_destination(*port as u16);
+            tcp_header.set_sequence(rand::random::<u32>());
+            tcp_header.set_acknowledgement(0);
+            tcp_header.set_data_offset(5);
+            tcp_header.set_reserved(0);
+            tcp_header.set_flags(TcpFlags::SYN);
+            tcp_header.set_window(64240);
+            tcp_header.set_urgent_ptr(0);
+            // tcp_header.set_options(&[TcpOption::mss(1460)]);
+
+            // Calculate checksum
+            let checksum = tcp::ipv4_checksum(
+                &tcp_header.to_immutable(),
+                &std_to_pnet_ipv4(&source_ip),
+                &std_to_pnet_ipv4(&target),
+            );
+            tcp_header.set_checksum(checksum);
+
+            match tx.send_to(tcp_header, *target) {
+                Ok(_) => {}
+                Err(e) => eprintln!("Failed to send packet: {}", e),
+            }
+
+            pb.inc(1);
+            thread::sleep(Duration::from_micros(100));
+        }
+    }
+
+    // Wait for receiver to finish
+    receiver_handle.join().unwrap();
+
+    // Convert results to the return format
+    let results_map = results.lock().unwrap();
+    targets
+        .iter()
+        .map(|ip| ScanResult {
+            ip: *ip,
+            open_ports: results_map.get(ip).cloned().unwrap_or_default(),
+        })
+        .collect()
+}