From 556e6650d001bdc7700813735dddb8d574d6a717 Mon Sep 17 00:00:00 2001 From: Michael Mikovsky <77305074+Astatin3@users.noreply.github.com> Date: Wed, 16 Apr 2025 21:46:23 -0600 Subject: [PATCH] Add cmdline tools, database, port scanning, and searching --- .gitignore | 1 + Cargo.toml | 8 +- src/database.rs | 624 ++++++++++++++++++++++++++++++++ src/lib.rs | 2 + src/main.rs | 205 ++++++++++- src/online_scan/online_scan.rs | 24 +- src/online_scan/ping_scanner.rs | 55 ++- src/parse_ip_range.rs | 4 + src/port_scan/mod.rs | 1 + src/port_scan/port_scan.rs | 37 ++ src/port_scan/tcp_scan.rs | 136 +++++++ src/ports.rs | 0 12 files changed, 1054 insertions(+), 43 deletions(-) create mode 100644 src/database.rs create mode 100644 src/ports.rs diff --git a/.gitignore b/.gitignore index 6985cf1..2bcf15e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ Cargo.lock # MSVC Windows builds of rustc generate these, which store debugging information *.pdb +ping_result_database/ diff --git a/Cargo.toml b/Cargo.toml index 7c50313..18a89bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,13 @@ version = "0.1.0" edition = "2024" [dependencies] -loadingbar = "1.0.1" +bincode = { version = "2.0.1", features = ["serde"] } +byteorder = "1.5.0" +indicatif = "0.17.11" +memchr = "2.7.4" pnet = "0.35.0" rand = "0.9.0" rocksdb = "0.23.0" +serde = { version = "1.0.219", features = ["derive"] } +serde_json = "1.0.140" +tokio = "1.44.2" diff --git a/src/database.rs b/src/database.rs new file mode 100644 index 0000000..aa9740b --- /dev/null +++ b/src/database.rs @@ -0,0 +1,624 @@ +use std::{ + collections::HashSet, + io::Error, + net::IpAddr, + sync::{Arc, Mutex}, + time::Instant, +}; + +use byteorder::{BigEndian, ByteOrder, WriteBytesExt}; +use memchr::memmem; +use rocksdb::{Cache, ColumnFamily, DB, IteratorMode, Options, ReadOptions, WriteBatch}; +use serde::{Deserialize, Serialize}; + +use crate::port_scan::port_scan::ScanResult; + +static COLUMN_COUNT: usize = 5; +static TEST_ROW_COUNT: usize = 1000; + +// Global settings for optimal performance +const BLOCK_CACHE_SIZE_MB: usize = 512; // 512MB block cache +const WRITE_BUFFER_SIZE_MB: usize = 64; // 64MB write buffer +const NUM_PARALLEL_THREADS: usize = 8; // Number of threads for parallel operations +const BATCH_SIZE: usize = 1000; // Batch size for writes + +pub struct ResultDatabase { + pub path: String, + options: Options, + columns: Vec, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct StringRow { + pub id: String, // Row identifier + pub values: Vec, // Array of string values +} + +impl StringRow { + pub fn to_string(&self) -> String { + let mut str = "".to_string(); + + str += format!("Row ID: {}, Values: [", self.id).as_str(); + for (i, value) in self.values.iter().enumerate() { + if i > 0 { + str += ", "; + } + str += format!("{}: \"{}\"", i, value).as_str(); + } + str += "]"; + + str + } +} + +/// Enum for defining search criteria +#[derive(Debug)] +enum SearchCriteria { + ByColumnValue(usize, String), // Search by specific column value + ByColumnPrefix(usize, String), // Search by column value prefix + // ByIdRange(String, String), // Search by ID range +} + +impl ResultDatabase { + pub fn new(path: &str) -> Self { + let mut options = Options::default(); + + options.create_if_missing(true); + options.create_missing_column_families(true); + options.increase_parallelism(NUM_PARALLEL_THREADS as i32); // Use multiple background threads + options.set_max_background_jobs(4); + options.set_write_buffer_size(WRITE_BUFFER_SIZE_MB * 1024 * 1024); // Larger write buffer + options.set_max_write_buffer_number(3); // Allow more write buffers + options.set_target_file_size_base(64 * 1024 * 1024); // 64MB per SST file + options.set_level_zero_file_num_compaction_trigger(4); // Start compaction after 4 L0 files + options.set_level_zero_slowdown_writes_trigger(16); // Start slowing down writes after 16 L0 files + options.set_level_zero_stop_writes_trigger(24); // Stop writes after 24 L0 files + options.set_max_bytes_for_level_base(512 * 1024 * 1024); // 512MB for base level + options.set_disable_auto_compactions(false); // Enable auto compactions + options.optimize_level_style_compaction(WRITE_BUFFER_SIZE_MB * 1024 * 1024); + options.set_max_total_wal_size(256 * 1024 * 1024); // 256MB max for WAL files + options.set_keep_log_file_num(5); // Keep 5 log files + options.set_log_level(rocksdb::LogLevel::Warn); // Minimal logging + + // Set up block cache for improved read performance + let mut block_opts = rocksdb::BlockBasedOptions::default(); + block_opts.set_block_cache(&Cache::new_lru_cache(BLOCK_CACHE_SIZE_MB * 1024 * 1024)); + block_opts.set_bloom_filter(10.0, false); + block_opts.set_whole_key_filtering(true); + block_opts.set_cache_index_and_filter_blocks(true); + block_opts.set_pin_l0_filter_and_index_blocks_in_cache(true); + options.set_block_based_table_factory(&block_opts); + + // Define column families for different indexes + + let mut column_families = vec!["default".to_string()]; // Main data store + + // Add column families for each column index we might want to search by + // (for demo, we'll create indexes for 5 potential columns) + for i in 0..COLUMN_COUNT { + column_families.push(format!("col{}_idx", i).to_string()); + } + + Self { + path: path.to_string(), + options, + columns: column_families, + } + } + + pub fn add_ping_results( + &self, + results: &Vec, + ) -> Result<(), Box> { + let mut string_rows = Vec::with_capacity(results.len()); // Pre-allocate capacity + + for result in results { + string_rows.push(StringRow { + id: result.to_string(), + values: vec![], + }); + } + + return self.save_rows(string_rows); + } + + pub fn add_tcp_results( + &self, + results: &Vec, + ) -> Result<(), Box> { + let mut string_rows = Vec::with_capacity(results.len()); // Pre-allocate capacity + + for result in results { + string_rows.push(result.to_string_row()); + } + + return self.save_rows(string_rows); + } + + pub fn save_rows(&self, string_rows: Vec) -> Result<(), Box> { + let db = Arc::new(DB::open_cf(&self.options, &self.path, &self.columns)?); + let cf_default = db.cf_handle("default").unwrap(); + + // Get handles to column index families + let mut cf_columns = Vec::new(); + for i in 0..3 { + let cf = db.cf_handle(&format!("col{}_idx", i)).unwrap(); + cf_columns.push(cf); + } + + let start = Instant::now(); + let length = string_rows.len(); + + // Split the rows into chunks for parallel processing + let chunks: Vec> = string_rows + .chunks(BATCH_SIZE) + .map(|chunk| chunk.to_vec()) + .collect(); + + // Process chunks in parallel + let elapsed = { + let db_ref = Arc::clone(&db); + let cf_default_ref = cf_default; + let cf_columns_ref = &cf_columns; + + // Create batches in parallel but write them sequentially + let batches: Vec = chunks + .into_iter() + .map(|chunk| { + let mut batch = WriteBatch::default(); + + for row in chunk { + // Use optimized binary format for the main data + let mut data = Vec::with_capacity(256); + + // Format: id_len + id + count + (len + str) for each value + // Binary format: direct encoding without JSON overhead + encode_row_binary(&mut data, &row); + + // Store in main column family + batch.put_cf(cf_default_ref, row.id.as_bytes(), &data); + + // Create indexes only for searchable columns (0-2) + for (col_idx, value) in row.values.iter().enumerate() { + if col_idx < cf_columns_ref.len() { + // Create search-friendly keys: value:rowid + // Use minimal escaping for better performance + let idx_key = format!("{}:{}", fast_escape(value), row.id); + batch.put_cf( + cf_columns_ref[col_idx], + idx_key.as_bytes(), + row.id.as_bytes(), + ); + } + } + } + + batch + }) + .collect(); + + // Write all batches to the database + for batch in batches { + db_ref.write(batch)?; + } + + // Force a flush to ensure all data is persisted + db_ref.flush()?; + + start.elapsed() + }; + + println!("Saved {} rows in {}ms", length, elapsed.as_millis()); + + Ok(()) + } + + pub fn get_row_by_host(&self, row: &str) -> Option { + let db = DB::open_cf(&self.options, &self.path, &self.columns); + if db.is_err() { + return None; + }; + let db = db.unwrap(); + let cf_default = db.cf_handle("default").unwrap(); + + return fetch_row(&db, cf_default, row); + } + + pub fn get_rows_by_port(&self, port: &str) -> Vec { + if let Ok(result) = self.search_substring_in_column(self.columns[0].as_str(), port) { + return result; + } else { + return Vec::new(); + } + } + + pub fn search_substring_in_column( + &self, + column: &str, + substring: &str, + ) -> Result, rocksdb::Error> { + let db = Arc::new(DB::open_cf(&self.options, &self.path, &self.columns)?); + + let cf = db.cf_handle(column).unwrap(); + + let mut matching_keys: Vec = Vec::new(); + + // Use RocksDB's iterator for efficient scanning + let iter = db.iterator_cf(cf, IteratorMode::Start); + + // Iterate through all key-value pairs in the column family + for item in iter { + let (key_bytes, value_bytes) = item?; + + // Convert value to string (assumes UTF-8 encoding) + if let Ok(value_str) = std::str::from_utf8(&value_bytes) { + // Check if the value contains the substring + if value_str.contains(substring) { + // Convert key to string and add to results + if let Ok(key_str) = std::str::from_utf8(&key_bytes) { + if let Some(row) = decode_row_binary(&value_bytes) { + matching_keys.push(row); + } + } + } + } + } + + Ok(matching_keys) + } +} + +// Count results from a search without printing +fn search( + db: &DB, + cf_default: &ColumnFamily, + cf_columns: &[&ColumnFamily], + criteria: SearchCriteria, +) -> Result, Box> { + let mut results: Vec = Vec::new(); + + match criteria { + SearchCriteria::ByColumnValue(col_idx, value) => { + if col_idx >= cf_columns.len() { + return Ok(results); + } + + // Create search key with escaped value + let prefix = format!("{}:", fast_escape(&value)); + let mut opts = ReadOptions::default(); + opts.set_prefix_same_as_start(true); + + let iterator = db.iterator_cf_opt( + cf_columns[col_idx], + opts, + rocksdb::IteratorMode::From(prefix.as_bytes(), rocksdb::Direction::Forward), + ); + + for item in iterator { + let (idx_key, data) = item?; + let idx_key_str = String::from_utf8(idx_key.to_vec())?; + + // Skip if we've moved past our prefix + if !idx_key_str.starts_with(&prefix) { + break; + } + + let row = decode_row_binary(&data); + + if let Some(row) = row { + results.push(row); + } + } + } + + SearchCriteria::ByColumnPrefix(col_idx, prefix) => { + if col_idx >= cf_columns.len() { + return Ok(results); + } + + // Create search key with escaped prefix + let search_prefix = fast_escape(&prefix); + + let iterator = db.iterator_cf( + cf_columns[col_idx], + rocksdb::IteratorMode::From(search_prefix.as_bytes(), rocksdb::Direction::Forward), + ); + + for item in iterator { + let (idx_key, data) = item?; + let idx_key_str = String::from_utf8(idx_key.to_vec())?; + + // Extract just the value part of the index key + let parts: Vec<&str> = idx_key_str.splitn(2, ':').collect(); + if parts.len() < 2 { + continue; + } + + let value_part = fast_unescape(parts[0]); + + // Skip if value doesn't start with our prefix + if !value_part.starts_with(&prefix) { + // If we've moved past potential matches, break early + if value_part > prefix { + break; + } + continue; + } + + let row = decode_row_binary(&data); + + if let Some(row) = row { + results.push(row); + } + } + } + } + + Ok(results) +} + +// Fast minimal escaping for key values +#[inline] +fn fast_escape(s: &str) -> String { + // Only escape the colon character which is our separator + s.replace(":", "\\:") +} + +// Fast unescaping for key values +#[inline] +fn fast_unescape(s: &str) -> String { + // Only unescape the colon + s.replace("\\:", ":") +} + +// Fast direct row fetch by ID +fn fetch_row(db: &DB, cf_default: &ColumnFamily, row_id: &str) -> Option { + match db.get_cf(cf_default, row_id.as_bytes()) { + Ok(Some(value)) => decode_row_binary(&value), + _ => None, + } +} + +// Fast column value fetch +fn fetch_column(db: &DB, cf_default: &ColumnFamily, row_id: &str, column_idx: usize) -> String { + match fetch_row(db, cf_default, row_id) { + Some(row) => get_column_value(&row, column_idx), + None => String::new(), + } +} + +// Get a column value, returning empty string if column doesn't exist +#[inline] +fn get_column_value(row: &StringRow, column_index: usize) -> String { + if column_index < row.values.len() { + row.values[column_index].clone() + } else { + String::new() // Return empty string for missing columns + } +} + +// Binary decoding of row data +fn decode_row_binary(data: &[u8]) -> Option { + if data.len() < 8 { + return None; + } + + let mut pos = 0; + + // Read ID length + let mut id_len_bytes = [0u8; 4]; + id_len_bytes.copy_from_slice(&data[pos..pos + 4]); + let id_len = u32::from_le_bytes(id_len_bytes) as usize; + pos += 4; + + // Read ID + if pos + id_len > data.len() { + return None; + } + let id = String::from_utf8_lossy(&data[pos..pos + id_len]).to_string(); + pos += id_len; + + // Read number of values + if pos + 4 > data.len() { + return None; + } + let mut values_count_bytes = [0u8; 4]; + values_count_bytes.copy_from_slice(&data[pos..pos + 4]); + let values_count = u32::from_le_bytes(values_count_bytes) as usize; + pos += 4; + + // Read values + let mut values = Vec::with_capacity(values_count); + for _ in 0..values_count { + if pos + 4 > data.len() { + return None; + } + + let mut value_len_bytes = [0u8; 4]; + value_len_bytes.copy_from_slice(&data[pos..pos + 4]); + let value_len = u32::from_le_bytes(value_len_bytes) as usize; + pos += 4; + + if pos + value_len > data.len() { + return None; + } + + let value = String::from_utf8_lossy(&data[pos..pos + value_len]).to_string(); + values.push(value); + pos += value_len; + } + + Some(StringRow { id, values }) +} + +// Binary encoding of row data for maximum performance +fn encode_row_binary(buf: &mut Vec, row: &StringRow) { + // Write ID length and ID + let id_bytes = row.id.as_bytes(); + buf.extend_from_slice(&(id_bytes.len() as u32).to_le_bytes()); + buf.extend_from_slice(id_bytes); + + // Write number of values + buf.extend_from_slice(&(row.values.len() as u32).to_le_bytes()); + + // Write each value + for value in &row.values { + let value_bytes = value.as_bytes(); + buf.extend_from_slice(&(value_bytes.len() as u32).to_le_bytes()); + buf.extend_from_slice(value_bytes); + } +} + +// fn benchmark_create_rows() { +// for i in 0..10000 { +// // Generate 10,000 test rows +// let mut values = Vec::with_capacity(5); + +// // Add IP address (column 0) +// if i % 3 == 0 { +// values.push(format!("192.168.1.{}", i % 255)); +// } else if i % 3 == 1 { +// values.push(format!("10.0.{}.{}", (i / 255) % 255, i % 255)); +// } else { +// values.push(format!("172.16.{}.{}", (i / 255) % 255, i % 255)); +// } + +// // Add status (column 1) +// if i % 5 < 4 { +// // 80% active +// values.push("active".to_string()); +// } else { +// values.push("inactive".to_string()); +// } + +// // Add response time (column 2) for active servers +// if i % 5 < 4 { +// values.push(format!("{}ms", (i % 100) + 1)); +// } + +// // Add server name (column 3) +// if i % 2 == 0 { +// values.push(format!("server{:04}", i)); +// } + +// // Add priority (column 4) for some servers +// if i % 7 == 0 { +// values.push("high_priority".to_string()); +// } else if i % 11 == 0 { +// values.push("low_priority".to_string()); +// } + +// // string_rows.push(StringRow { +// // id: format!("row{:06}", i), +// // values, +// // }); +// } +// } + +// // Benchmark search performance +// fn benchmark_search( +// db: &DB, +// cf_default: &ColumnFamily, +// cf_columns: &[&ColumnFamily], +// name: &str, +// criteria_fn: F, +// ) -> Result<(), Box> +// where +// F: Fn() -> SearchCriteria, +// { +// let mut total_duration = Duration::from_secs(0); +// let mut total_results = 0; + +// for i in 1..=3 { +// let criteria = criteria_fn(); +// let start = Instant::now(); +// let count = count_search_results(db, cf_default, cf_columns, criteria)?; +// let duration = start.elapsed(); + +// total_duration += duration; +// total_results = count; // All runs should return same count + +// println!(" Run {}: Found {} results in {:?}", i, count, duration); +// } + +// let avg_duration = total_duration / 3; +// println!( +// " Average: {:?} for {} results", +// avg_duration, total_results +// ); +// println!( +// " Speed: {:.2} results/ms", +// total_results as f64 / avg_duration.as_millis() as f64 +// ); + +// Ok(()) +// } + +// // Benchmark direct row fetch performance +// fn benchmark_direct_fetch( +// db: &DB, +// cf_default: &ColumnFamily, +// name: &str, +// row_id: &str, +// ) -> Result<(), Box> { +// let mut total_duration = Duration::from_secs(0); + +// for i in 1..=3 { +// let start = Instant::now(); + +// // Do multiple fetches to get a measurable time +// for _ in 0..1000 { +// let _ = fetch_row(db, cf_default, row_id); +// } + +// let duration = start.elapsed(); +// total_duration += duration; + +// println!(" Run {}: 1000 row fetches in {:?}", i, duration); +// } + +// let avg_duration = total_duration / 3; +// println!(" Average: {:?} for 1000 fetches", avg_duration); +// println!( +// " Speed: {:.2} fetches/ms", +// 1000.0 / avg_duration.as_millis() as f64 +// ); + +// Ok(()) +// } + +// // Benchmark column fetch performance +// fn benchmark_column_fetch( +// db: &DB, +// cf_default: &ColumnFamily, +// name: &str, +// row_id: &str, +// col_idx: usize, +// ) -> Result<(), Box> { +// let mut total_duration = Duration::from_secs(0); + +// for i in 1..=3 { +// let start = Instant::now(); + +// // Do multiple fetches to get a measurable time +// for _ in 0..1000 { +// let _ = fetch_column(db, cf_default, row_id, col_idx); +// } + +// let duration = start.elapsed(); +// total_duration += duration; + +// println!(" Run {}: 1000 column fetches in {:?}", i, duration); +// } + +// let avg_duration = total_duration / 3; +// println!(" Average: {:?} for 1000 fetches", avg_duration); +// println!( +// " Speed: {:.2} fetches/ms", +// 1000.0 / avg_duration.as_millis() as f64 +// ); + +// Ok(()) +// } +// +// // Example usage with batching for very large datasets diff --git a/src/lib.rs b/src/lib.rs index a605ff7..cd994da 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,2 +1,4 @@ +pub mod database; pub mod online_scan; pub mod parse_ip_range; +pub mod port_scan; diff --git a/src/main.rs b/src/main.rs index a7f44af..54cc4e4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,28 +1,207 @@ -pub mod online_scan; -pub mod parse_ip_range; +use std::{env, net::IpAddr, time::Duration}; -use std::env; - -use online_scan::ping_scanner; +use online_scan::PingResult; use parse_ip_range::parse_ip_targets; +use untitled::{database::ResultDatabase, online_scan, parse_ip_range, port_scan::tcp_scan}; fn main() -> Result<(), Box> { let args: Vec = env::args().collect(); + let database = ResultDatabase::new("ping_result_database"); + if args.len() <= 1 { + println!("You must specify a command!"); + print_help(None); + } + + match args[1].to_lowercase().as_str() { + "scan" => { + if args.len() != 4 { + println!("Invalid Usage!"); + print_help(Some(args[1].as_str())); + return Ok(()); + } + let _ = scan(database, args[2].clone(), args[3].clone()); + } + "search" => { + if args.len() != 4 { + println!("Invalid Usage!"); + print_help(Some(args[1].as_str())); + return Ok(()); + } + search(database, args[2].to_string(), args[3].to_string()); + } + "help" => { + if args.len() != 3 { + print_help(None); + return Ok(()); + } + print_help(Some(args[2].as_str())); + } + _ => { + println!("Invalid command!"); + print_help(None); + return Ok(()); + } + } + + Ok(()) +} +fn scan( + database: ResultDatabase, + search_type: String, + arg: String, +) -> Result<(), Box> { // Set default targets or use command line input - let targets = if args.len() > 1 { - args[1].clone() - } else { - "".to_string() - }; + let targets = arg; // Parse the targets into IP addresses let hosts = parse_ip_targets(&targets)?; - let length = hosts.len(); + match search_type.as_str() { + "ping" => { + let length = hosts.len(); - let results = ping_scanner::ping_scan(hosts).unwrap(); + let up_hosts: Vec = online_scan::ping_scanner::ping_scan(hosts).unwrap(); + println!("Finished! {} Scanned, {} Up", length, up_hosts.len()); + let _ = database.add_ping_results(&up_hosts); + } + "tcp" => { + let length = hosts.len(); + + let up_hosts: Vec = online_scan::ping_scanner::ping_scan(hosts).unwrap(); + println!("Finished! {} Scanned, {} Up", length, up_hosts.len()); + let _ = database.add_ping_results(&up_hosts); + + let tcp_ports = vec![ + 1, 3, 4, 6, 7, 9, 13, 17, 19, 20, 21, 22, 23, 24, 25, 26, 30, 32, 33, 37, 42, 43, + 49, 53, 70, 79, 80, 81, 82, 83, 84, 85, 88, 89, 90, 99, 100, 106, 109, 110, 111, + 113, 119, 125, 135, 139, 143, 144, 146, 161, 163, 179, 199, 211, 212, 222, 254, + 255, 256, 259, 264, 280, 301, 306, 311, 340, 366, 389, 406, 407, 416, 417, 425, + 427, 443, 444, 445, 458, 464, 465, 481, 497, 500, 512, 513, 514, 515, 524, 541, + 543, 544, 545, 548, 554, 555, 563, 587, 593, 616, 617, 625, 631, 636, 646, 648, + 666, 667, 668, 683, 687, 691, 700, 705, 711, 714, 720, 722, 726, 749, 765, 777, + 783, 787, 800, 801, 808, 843, 873, 880, 888, 898, 900, 901, 902, 903, 911, 912, + 981, 987, 990, 992, 993, 995, 999, 1000, 1001, 1002, 1007, 1009, 1010, 1011, 1021, + 1022, 1023, 1024, 1025, 1026, 1027, 1028, 1029, 1030, 1031, 1032, 1033, 1034, 1035, + 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047, 1048, 1049, + 1050, 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, 1059, 1060, 1061, 1062, 1063, + 1064, 1065, 1066, 1067, 1068, 1069, 1070, 1071, 1072, 1073, 1074, 1075, 1076, 1077, + 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, 1088, 1089, 1090, 1091, + 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1100, 1102, 1104, 1105, 1106, 1107, + 1108, 1110, 1111, 1112, 1113, 1114, 1117, 1119, 1121, 1122, 1123, 1124, 1126, 1130, + 1131, 1132, 1137, 1138, 1141, 1145, 1147, 1148, 1149, 1151, 1152, 1154, 1163, 1164, + 1165, 1166, 1169, 1174, 1175, 1183, 1185, 1186, 1187, 1192, 1198, 1199, 1201, 1213, + 1216, 1217, 1218, 1233, 1234, 1236, 1244, 1247, 1248, 1259, 1271, 1272, 1277, 1287, + 1296, 1300, 1301, 1309, 1310, 1311, 1322, 1328, 1334, 1352, 1417, 1433, 1434, 1443, + 1455, 1461, 1494, 1500, 1501, 1503, 1521, 1524, 1533, 1556, 1580, 1583, 1594, 1600, + 1641, 1658, 1666, 1687, 1688, 1700, 1717, 1718, 1719, 1720, 1721, 1723, 1755, 1761, + 1782, 1783, 1801, 1805, 1812, 1839, 1840, 1862, 1863, 1864, 1875, 1900, 1914, 1935, + 1947, 1971, 1972, 1974, 1984, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, + 2007, 2008, 2009, 2010, 2013, 2020, 2021, 2022, 2030, 2033, 2034, 2035, 2038, 2040, + 2041, 2042, 2043, 2045, 2046, 2047, 2048, 2049, 2065, 2068, 2099, 2100, 2103, 2105, + 2106, 2107, 2111, 2119, 2121, 2126, 2135, 2144, 2160, 2161, 2170, 2179, 2190, 2191, + 2196, 2200, 2222, 2251, 2260, 2288, 2301, 2323, 2366, 2381, 2382, 2383, 2393, 2394, + 2399, 2401, 2492, 2500, 2522, 2525, 2557, 2601, 2602, 2604, 2605, 2607, 2608, 2638, + 2701, 2702, 2710, 2717, 2718, 2725, 2800, 2809, 2811, 2869, 2875, 2909, 2910, 2920, + 2967, 2968, 2998, 3000, 3001, 3003, 3005, 3006, 3007, 3011, 3013, 3017, 3030, 3031, + 3052, 3071, 3077, 3128, 3168, 3211, 3221, 3260, 3261, 3268, 3269, 3283, 3300, 3301, + 3306, 3322, 3323, 3324, 3325, 3333, 3351, 3367, 3369, 3370, 3371, 3372, 3389, 3390, + 3404, 3476, 3493, 3517, 3527, 3546, 3551, 3580, 3659, 3689, 3690, 3703, 3737, 3766, + 3784, 3800, 3801, 3809, 3814, 3826, 3827, 3828, 3851, 3869, 3871, 3878, 3880, 3889, + 3905, 3914, 3918, 3920, 3945, 3971, 3986, 3995, 3998, 4000, 4001, 4002, 4003, 4004, + 4005, 4006, 4045, 4111, 4125, 4126, 4129, 4224, 4242, 4279, 4321, 4343, 4443, 4444, + 4445, 4446, 4449, 4550, 4567, 4662, 4848, 4899, 4900, 4998, 5000, 5001, 5002, 5003, + 5004, 5009, 5030, 5033, 5050, 5051, 5054, 5060, 5061, 5080, 5087, 5100, 5101, 5102, + 5120, 5190, 5200, 5214, 5221, 5222, 5225, 5226, 5269, 5280, 5298, 5357, 5405, 5414, + 5431, 5432, 5440, 5500, 5510, 5544, 5550, 5555, 5560, 5566, 5631, 5633, 5666, 5678, + 5679, 5718, 5730, 5800, 5801, 5802, 5810, 5811, 5815, 5822, 5825, 5850, 5859, 5862, + 5877, 5900, 5901, 5902, 5903, 5904, 5906, 5907, 5910, 5911, 5915, 5922, 5925, 5950, + 5952, 5959, 5960, 5961, 5962, 5963, 5987, 5988, 5989, 5998, 5999, 6000, 6001, 6002, + 6003, 6004, 6005, 6006, 6007, 6009, 6025, 6059, 6100, 6101, 6106, 6112, 6123, 6129, + 6156, 6346, 6389, 6502, 6510, 6543, 6547, 6565, 6566, 6567, 6580, 6646, 6666, 6667, + 6668, 6669, 6689, 6692, 6699, 6779, 6788, 6789, 6792, 6839, 6881, 6901, 6969, 7000, + 7001, 7002, 7004, 7007, 7019, 7025, 7070, 7100, 7103, 7106, 7200, 7201, 7402, 7435, + 7443, 7496, 7512, 7625, 7627, 7676, 7741, 7777, 7778, 7800, 7911, 7920, 7921, 7937, + 7938, 7999, 8000, 8001, 8002, 8007, 8008, 8009, 8010, 8011, 8021, 8022, 8031, 8042, + 8045, 8080, 8081, 8082, 8083, 8084, 8085, 8086, 8087, 8088, 8089, 8090, 8093, 8099, + 8100, 8180, 8181, 8192, 8193, 8194, 8200, 8222, 8254, 8290, 8291, 8292, 8300, 8333, + 8383, 8400, 8402, 8443, 8500, 8600, 8649, 8651, 8652, 8654, 8701, 8800, 8873, 8888, + 8899, 8994, 9000, 9001, 9002, 9003, 9009, 9010, 9011, 9040, 9050, 9071, 9080, 9081, + 9090, 9091, 9099, 9100, 9101, 9102, 9103, 9110, 9111, 9200, 9207, 9220, 9290, 9415, + 9418, 9485, 9500, 9502, 9503, 9535, 9575, 9593, 9594, 9595, 9618, 9666, 9876, 9877, + 9878, 9898, 9900, 9917, 9929, 9943, 9944, 9968, 9998, 9999, 10000, 10001, 10002, + 10003, 10004, 10009, 10010, 10012, 10024, 10025, 10082, 10180, 10215, 10243, 10566, + 10616, 10617, 10621, 10626, 10628, 10629, 10778, 11110, 11111, 11967, 12000, 12174, + 12265, 12345, 13456, 13722, 13782, 13783, 14000, 14238, 14441, 14442, 15000, 15002, + 15003, 15004, 15660, 15742, 16000, 16001, 16012, 16016, 16018, 16080, 16113, 16992, + 16993, 17877, 17988, 18040, 18101, 18988, 19101, 19283, 19315, 19350, 19780, 19801, + 19842, 20000, 20005, 20031, 20221, 20222, 20828, 21571, 22939, 23502, 24444, 24800, + 25734, 25735, 26214, 27000, 27352, 27353, 27355, 27356, 27715, 28201, 30000, 30718, + 30951, 31038, 31337, 32768, 32769, 32770, 32771, 32772, 32773, 32774, 32775, 32776, + 32777, 32778, 32779, 32780, 32781, 32782, 32783, 32784, 32785, 33354, 33899, 34571, + 34572, 34573, 35500, 38292, 40193, 40911, 41511, 42510, 44176, 44442, 44443, 44501, + 45100, 48080, 49152, 49153, 49154, 49155, 49156, 49157, 49158, 49159, 49160, 49161, + 49163, 49165, 49167, 49175, 49176, 49400, 49999, 50000, 50001, 50002, 50003, 50006, + 50300, 50389, 50500, 50636, 50800, 51103, 51493, 52673, 52822, 52848, 52869, 54045, + 54328, 55055, 55056, 55555, 55600, 56737, 56738, 57294, 57797, 58080, 60020, 60443, + 61532, 61900, 62078, 63331, 64623, 64680, 65000, 65129, 65389, + ]; + + let tcp_results = tcp_scan::tcp_scan(up_hosts, tcp_ports, Duration::from_secs(3)); + let _ = database.add_tcp_results(&tcp_results); + } + _ => { + println!("Invalid search type!"); + } + } - println!("Finished! {} Scanned, {} Up", length, results.len()); Ok(()) } + +fn search(database: ResultDatabase, search_type: String, arg: String) { + match search_type.as_str() { + "host" => { + let row = database.get_row_by_host(&arg); + if let Some(row) = row { + println!("{}", row.to_string()); + } else { + println!("Could not find host by argument {}", arg.as_str()); + } + } + + "port" => { + let rows = database.get_rows_by_port(&arg); + + for row in rows { + println!("{}", row.to_string()); + } + } + _ => { + println!("Invalid search type!"); + } + } +} + +fn print_help(arg: Option<&str>) { + println!( + "{}", + match arg { + None => { + "rust-scan help menu +Commands: + scan - scan a block of addresses and check for online using icmp echo + search - Search database + help (command) - Print help" + } + Some("pingscan") => { + "pingscan +scan a block of addresses and check for online using icmp echo +Usage: pingscan 10.42.0.1,12.34.0.0-12.34.56.78,127.0.0.0/8" + } + Some(_) => { + print_help(None); + "" + } + } + ); +} diff --git a/src/online_scan/online_scan.rs b/src/online_scan/online_scan.rs index 9cb8332..fa9e04a 100644 --- a/src/online_scan/online_scan.rs +++ b/src/online_scan/online_scan.rs @@ -1,7 +1,11 @@ use std::{net::IpAddr, time::Duration}; +use serde::{Deserialize, Serialize}; + +use crate::database::StringRow; + // Structure to hold ping results -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct PingResult { pub host: IpAddr, pub is_up: bool, @@ -16,4 +20,22 @@ impl PingResult { response_time: None, } } + + pub fn to_string_row(&self) -> StringRow { + StringRow { + id: self.host.to_string(), + values: vec![ + if self.is_up { + "up".to_string() + } else { + "down".to_string() + }, + if self.response_time.is_some() { + self.response_time.unwrap().as_millis().to_string() + } else { + "None".to_string() + }, + ], + } + } } diff --git a/src/online_scan/ping_scanner.rs b/src/online_scan/ping_scanner.rs index ee60c80..79ffa6e 100644 --- a/src/online_scan/ping_scanner.rs +++ b/src/online_scan/ping_scanner.rs @@ -1,5 +1,5 @@ +use indicatif::ProgressBar; use pnet::packet::ip::IpNextHeaderProtocols; -use pnet::packet::ipv4::Ipv4OptionNumbers::TR; use pnet::packet::{ Packet, icmp::{IcmpTypes, echo_request::MutableEchoRequestPacket}, @@ -8,7 +8,6 @@ use pnet::transport::{ TransportChannelType, TransportProtocol, icmp_packet_iter, transport_channel, }; use pnet::util::checksum; -use std::cmp::max; use std::collections::HashMap; use std::net::IpAddr; use std::sync::atomic::{AtomicBool, Ordering}; @@ -18,12 +17,12 @@ use std::time::{Duration, Instant}; static TIMEOUT: Duration = Duration::from_secs(3); // static MAX_PINGS_PER_SECOND: u64 = 10000; -static SEND_DELAY_NANOS: Duration = Duration::from_nanos(500); +static SEND_DELAY_NANOS: Duration = Duration::from_micros(10); use crate::online_scan::PingResult; -pub fn ping_scan(hosts: Vec) -> Result, Box> { - let results = Arc::new(Mutex::new(Vec::::new())); +pub fn ping_scan(hosts: Vec) -> Result, Box> { + let results = Arc::new(Mutex::new(Vec::::new())); // Create a receiver channel for ICMP packets let (_, mut rx) = transport_channel( @@ -45,17 +44,24 @@ pub fn ping_scan(hosts: Vec) -> Result, Box = None; + // let mut pb: Option = None; // Keep receiving until timeout or all hosts are accounted for loop { // Stop reciving loop if timeout is reached // let time = finished_sending_time; - if finish_sending_time.is_some() && finish_sending_time.unwrap().elapsed() >= TIMEOUT { - break; + if finish_sending_time.is_some() { + let delay = finish_sending_time.unwrap().elapsed(); + // pb.as_ref().unwrap().set_position(delay.as_millis() as u64); + if delay >= TIMEOUT { + // pb.unwrap().finish_and_clear(); + break; + } } else if finish_sending_time.is_none() && recv_finished_sending_time.load(Ordering::Relaxed) { finish_sending_time = Some(Instant::now()); + // pb = Some(ProgressBar::new(TIMEOUT.as_millis() as u64)); println!("Waiting {} seconds for timeout...", TIMEOUT.as_secs()) } // if time.is_some() { @@ -65,7 +71,7 @@ pub fn ping_scan(hosts: Vec) -> Result, Box { if packet.get_icmp_type() == IcmpTypes::EchoReply { let payload = packet.payload(); @@ -79,11 +85,12 @@ pub fn ping_scan(hosts: Vec) -> Result, Box) -> Result, Box) -> Result, Box {} - Err(_) => { - let mut results = sender_results.lock().unwrap(); - results.push(PingResult { - host: host_clone, - is_up: false, - response_time: None, - }); - } - } + let _ = send_ping(host_clone, identifier); // let now = Instant::now(); // let delay = MAX_RATE_NANOS - last_send_time.duration_since(now).as_nanos() as u64; // last_send_time = now; + if (i % 16) == 0 { + pb.inc(16); + } thread::sleep(SEND_DELAY_NANOS); } - println!("Finished Sending!"); + pb.finish_and_clear(); sender_finished_sending_time.swap(true, Ordering::Relaxed); }); diff --git a/src/parse_ip_range.rs b/src/parse_ip_range.rs index b5897cf..dc70aef 100644 --- a/src/parse_ip_range.rs +++ b/src/parse_ip_range.rs @@ -3,6 +3,8 @@ use std::{ str::FromStr, }; +use rand::{rng, seq::SliceRandom}; + // static MAX_HOSTS: u32 = 1024; /// Parse a comma-separated list of IP targets @@ -30,6 +32,8 @@ pub fn parse_ip_targets(targets: &str) -> Result, Box, + // pub data: HashMap>, +} + +impl ScanResult { + pub fn new(ip: IpAddr) -> Self { + ScanResult { + ip, + open_ports: Vec::new(), + // data: HashMap::new(), + } + } + pub fn to_string_row(&self) -> StringRow { + StringRow { + id: self.ip.to_string(), + values: vec![join_nums(&self.open_ports, ",")], + } + } +} + +fn join_nums(nums: &Vec, sep: &str) -> String { + // 1. Convert numbers to strings + let str_nums: Vec = nums + .iter() + .map(|n| n.to_string()) // map every integer to a string + .collect(); // collect the strings into the vector + + // 2. Join the strings. There's already a function for this. + str_nums.join(sep) +} diff --git a/src/port_scan/tcp_scan.rs b/src/port_scan/tcp_scan.rs index 8b13789..a1b1c9f 100644 --- a/src/port_scan/tcp_scan.rs +++ b/src/port_scan/tcp_scan.rs @@ -1 +1,137 @@ +use std::collections::HashMap; +use std::net::{IpAddr, Ipv4Addr}; +use std::str::FromStr; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::Duration; +use indicatif::ProgressBar; +use pnet::datalink::{self, NetworkInterface}; +use pnet::packet::ip::IpNextHeaderProtocols; +use pnet::packet::tcp::{MutableTcpPacket, TcpFlags, TcpOption, TcpPacket}; +use pnet::packet::{Packet, tcp}; +use pnet::transport::{self, TransportChannelType}; +use pnet::util::checksum; +use rand::{random_range, random_ratio}; + +use super::port_scan::ScanResult; + +fn std_to_pnet_ipv4(previous: &IpAddr) -> Ipv4Addr { + Ipv4Addr::from_str(previous.to_string().as_str()).unwrap() +} + +// Main scanning function +pub fn tcp_scan(targets: Vec, ports: Vec, timeout: Duration) -> Vec { + // Find network interface + let interface = datalink::interfaces() + .into_iter() + .find(|iface| iface.is_up() && !iface.is_loopback() && !iface.ips.is_empty()) + .expect("No valid network interface found"); + + // Create transport channel for sending and receiving + let (mut tx, mut rx) = transport::transport_channel( + 65535, + TransportChannelType::Layer4(pnet::transport::TransportProtocol::Ipv4( + IpNextHeaderProtocols::Tcp, + )), + ) + .expect("Failed to create transport channel"); + + // Shared results + let results = Arc::new(Mutex::new(HashMap::>::new())); + + // Initialize results map + { + let mut results_map = results.lock().unwrap(); + for ip in &targets { + results_map.insert(*ip, Vec::new()); + } + } + + let receiver_results = Arc::clone(&results); + let receiver_handle = thread::spawn(move || { + let start_time = std::time::Instant::now(); + + while start_time.elapsed() < timeout { + let mut iter = transport::tcp_packet_iter(&mut rx); + + match iter.next() { + Ok((packet, addr)) => { + if let Some(tcp) = TcpPacket::new(packet.packet()) { + // Check for SYN+ACK flags (indicating open port) + if tcp.get_flags() == TcpFlags::SYN | TcpFlags::ACK { + let mut results_map = receiver_results.lock().unwrap(); + if let Some(open_ports) = results_map.get_mut(&addr) { + open_ports.push(tcp.get_source() as i32); + } + } + } + } + Err(_) => { + // Just continue on errors + thread::sleep(Duration::from_millis(1)); + } + } + } + }); + + let pb = ProgressBar::new((targets.len() * ports.len()) as u64); + + let source_ip = interface + .ips + .iter() + .find(|ip| ip.is_ipv4()) + .expect("No IPv4 address found") + .ip(); + + for target in &targets { + for port in &ports { + // let source_ip = Ipv4Addr::from_bits(random_range(0..=(0xffffffff))); + let source_port: u16 = random_range(1..=65535); + // println!("{}", source_ip.to_string()); + + let mut tcp_buffer = vec![0u8; 20 + 20]; // IP header + TCP header + let mut tcp_header = MutableTcpPacket::new(&mut tcp_buffer[0..]).unwrap(); + + tcp_header.set_source(source_port); + tcp_header.set_destination(*port as u16); + tcp_header.set_sequence(rand::random::()); + tcp_header.set_acknowledgement(0); + tcp_header.set_data_offset(5); + tcp_header.set_reserved(0); + tcp_header.set_flags(TcpFlags::SYN); + tcp_header.set_window(64240); + tcp_header.set_urgent_ptr(0); + // tcp_header.set_options(&[TcpOption::mss(1460)]); + + // Calculate checksum + let checksum = tcp::ipv4_checksum( + &tcp_header.to_immutable(), + &std_to_pnet_ipv4(&source_ip), + &std_to_pnet_ipv4(&target), + ); + tcp_header.set_checksum(checksum); + + match tx.send_to(tcp_header, *target) { + Ok(_) => {} + Err(e) => eprintln!("Failed to send packet: {}", e), + } + + pb.inc(1); + thread::sleep(Duration::from_micros(100)); + } + } + + // Wait for receiver to finish + receiver_handle.join().unwrap(); + + // Convert results to the return format + let results_map = results.lock().unwrap(); + targets + .iter() + .map(|ip| ScanResult { + ip: *ip, + open_ports: results_map.get(ip).cloned().unwrap_or_default(), + }) + .collect() +} diff --git a/src/ports.rs b/src/ports.rs new file mode 100644 index 0000000..e69de29