diff --git a/Cargo.toml b/Cargo.toml index 3e82b89..24ee20a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,3 +20,4 @@ craftping = "0.7.0" sha256 = "1.6.0" rayon = "1.10.0" futures = "0.3.31" +chrono = "0.4.40" diff --git a/src/database.rs b/src/database.rs index 9e89b21..7bcd650 100644 --- a/src/database.rs +++ b/src/database.rs @@ -1,5 +1,11 @@ -use std::{collections::HashMap, net::IpAddr, sync::Arc, time::Instant}; +use std::{ + collections::HashMap, + net::IpAddr, + sync::Arc, + time::{Duration, Instant, UNIX_EPOCH}, +}; +use chrono::{DateTime, Utc}; use regex::Regex; use rocksdb::{Cache, ColumnFamily, DB, IteratorMode, Options, WriteBatch}; use serde::{Deserialize, Serialize}; @@ -11,6 +17,7 @@ const BLOCK_CACHE_SIZE_MB: usize = 512; // 512MB block cache const WRITE_BUFFER_SIZE_MB: usize = 64; // 64MB write buffer const NUM_PARALLEL_THREADS: usize = 8; // Number of threads for parallel operations const BATCH_SIZE: usize = 1000; // Batch size for writes +pub const EPOCH_2025: u32 = 1735689600; // So i don't have to use u64 pub struct ResultDatabase { pub path: String, @@ -22,6 +29,7 @@ pub struct ResultDatabase { pub struct DatabaseResult { pub ip: String, pub port: u16, + pub time_scanned: u32, pub version: String, pub protocol: u32, @@ -45,9 +53,10 @@ impl DatabaseResult { let mut str = "".to_string(); str += format!( - "\n{}\n- ports: [{}]\n- version: [{}]\n- protocol: [{}]\n- max_players: [{}]\n- online_players: [{}]\n- players_list: [{:?}]\n- description: [{}]\n- icon_hash: [{}]\n- mod_info: [{:?}]\n- forge_data: [{:?}]\n- enforces_secure_chat: [{:?}]\n- previews_chat: [{:?}]", + "\n{}:{}\n- Last scanned: {}\n- version: [{}]\n- protocol: [{}]\n- max_players: [{}]\n- online_players: [{}]\n- players_list: [{:?}]\n- description: [{}]\n- icon_hash: [{}]\n- mod_info: [{:?}]\n- forge_data: [{:?}]\n- enforces_secure_chat: [{:?}]\n- previews_chat: [{:?}]", self.ip, self.port, + DateTime::::from(UNIX_EPOCH + Duration::from_secs((self.time_scanned + EPOCH_2025) as u64)).format("%Y-%m-%d %H:%M:%S.%f").to_string(), self.version, self.protocol, self.max_players, @@ -163,6 +172,7 @@ impl ResultDatabase { let column_families = vec![ "addr".to_string(), + "time".to_string(), "version".to_string(), "protocol".to_string(), "max_players".to_string(), @@ -245,17 +255,18 @@ impl ResultDatabase { let db = Arc::new(DB::open_cf(&self.options, &self.path, &self.columns)?); let cf_addr = db.cf_handle(&self.columns[0]).unwrap(); - let cf_version = db.cf_handle(&self.columns[1]).unwrap(); - let cf_protocol = db.cf_handle(&self.columns[2]).unwrap(); - let cf_max_players = db.cf_handle(&self.columns[3]).unwrap(); - let cf_online_players = db.cf_handle(&self.columns[4]).unwrap(); - let cf_players_list = db.cf_handle(&self.columns[5]).unwrap(); - let cf_description = db.cf_handle(&self.columns[6]).unwrap(); - let cf_icon_hash = db.cf_handle(&self.columns[7]).unwrap(); - let cf_mod_info = db.cf_handle(&self.columns[8]).unwrap(); - let cf_forge_data = db.cf_handle(&self.columns[9]).unwrap(); - let cf_enforces_secure_chat = db.cf_handle(&self.columns[10]).unwrap(); - let cf_previews_chat = db.cf_handle(&self.columns[11]).unwrap(); + let cf_time = db.cf_handle(&self.columns[1]).unwrap(); + let cf_version = db.cf_handle(&self.columns[2]).unwrap(); + let cf_protocol = db.cf_handle(&self.columns[3]).unwrap(); + let cf_max_players = db.cf_handle(&self.columns[4]).unwrap(); + let cf_online_players = db.cf_handle(&self.columns[5]).unwrap(); + let cf_players_list = db.cf_handle(&self.columns[6]).unwrap(); + let cf_description = db.cf_handle(&self.columns[7]).unwrap(); + let cf_icon_hash = db.cf_handle(&self.columns[8]).unwrap(); + let cf_mod_info = db.cf_handle(&self.columns[9]).unwrap(); + let cf_forge_data = db.cf_handle(&self.columns[10]).unwrap(); + let cf_enforces_secure_chat = db.cf_handle(&self.columns[11]).unwrap(); + let cf_previews_chat = db.cf_handle(&self.columns[12]).unwrap(); let start = Instant::now(); let length = string_rows.len(); @@ -282,6 +293,7 @@ impl ResultDatabase { let key = key.as_bytes(); batch.put_cf(cf_addr, key, key); + batch.put_cf(cf_time, key, row.time_scanned.to_string().as_bytes()); batch.put_cf(cf_version, key, row.version.as_bytes()); batch.put_cf(cf_protocol, key, row.protocol.to_string().as_bytes()); batch.put_cf(cf_max_players, key, row.max_players.to_string().as_bytes()); @@ -368,6 +380,7 @@ impl ResultDatabase { db.cf_handle(&self.columns[9]).unwrap(), db.cf_handle(&self.columns[10]).unwrap(), db.cf_handle(&self.columns[11]).unwrap(), + db.cf_handle(&self.columns[12]).unwrap(), ]; return self.fetch_row(&db, row, &cfs); @@ -413,6 +426,7 @@ impl ResultDatabase { db.cf_handle(&self.columns[9]).unwrap(), db.cf_handle(&self.columns[10]).unwrap(), db.cf_handle(&self.columns[11]).unwrap(), + db.cf_handle(&self.columns[12]).unwrap(), ]; let mut matching_keys: Vec = Vec::new(); @@ -457,6 +471,7 @@ impl ResultDatabase { db.cf_handle(&self.columns[9]).unwrap(), db.cf_handle(&self.columns[10]).unwrap(), db.cf_handle(&self.columns[11]).unwrap(), + db.cf_handle(&self.columns[12]).unwrap(), ]; let mut matching_keys: Vec = Vec::new(); @@ -490,7 +505,7 @@ impl ResultDatabase { if queries.len() == 1 { // Return host if results include host match queries[0] { - QueryDataType::Host(row, port) => { + QueryDataType::Addr(row, port) => { return Ok(vec![ self.get_row_by_host( format!("{}:{}", row.to_string().as_str(), port).as_str(), @@ -517,6 +532,7 @@ impl ResultDatabase { db.cf_handle(&self.columns[9]).unwrap(), db.cf_handle(&self.columns[10]).unwrap(), db.cf_handle(&self.columns[11]).unwrap(), + db.cf_handle(&self.columns[12]).unwrap(), ]; let matching_key_bytes = search_parallel(&db, queries, &cfs); @@ -545,34 +561,38 @@ impl ResultDatabase { .to_string() .parse::() .unwrap(), - version: self.row_to_string(db, row_id, &cfs[1]), - protocol: self - .row_to_string(db, row_id, &cfs[2]) + time_scanned: self + .row_to_string(db, row_id, &cfs[1]) .parse::() .unwrap(), - max_players: self + version: self.row_to_string(db, row_id, &cfs[2]), + protocol: self .row_to_string(db, row_id, &cfs[3]) .parse::() .unwrap(), - online_players: self + max_players: self .row_to_string(db, row_id, &cfs[4]) .parse::() .unwrap(), + online_players: self + .row_to_string(db, row_id, &cfs[5]) + .parse::() + .unwrap(), players_list: DatabaseResult::decode_players_list( - self.row_to_string(db, row_id, &cfs[5]), + self.row_to_string(db, row_id, &cfs[6]), ), - description: self.row_to_string(db, row_id, &cfs[6]), - icon_hash: self.row_to_string(db, row_id, &cfs[7]), - mod_info: DatabaseResult::decode_mod_info(self.row_to_string(db, row_id, &cfs[8])), + description: self.row_to_string(db, row_id, &cfs[7]), + icon_hash: self.row_to_string(db, row_id, &cfs[8]), + mod_info: DatabaseResult::decode_mod_info(self.row_to_string(db, row_id, &cfs[9])), forge_data: DatabaseResult::decode_forge_data( - self.row_to_string(db, row_id, &cfs[9]), - ), - enforces_secure_chat: DatabaseResult::decode_option_bool( self.row_to_string(db, row_id, &cfs[10]), ), - previews_chat: DatabaseResult::decode_option_bool( + enforces_secure_chat: DatabaseResult::decode_option_bool( self.row_to_string(db, row_id, &cfs[11]), ), + previews_chat: DatabaseResult::decode_option_bool( + self.row_to_string(db, row_id, &cfs[12]), + ), }), _ => None, } @@ -589,7 +609,10 @@ impl ResultDatabase { #[derive(Debug)] pub enum QueryDataType { - Host(IpAddr, u16), + Addr(IpAddr, u16), + Host(QueryType, String), + Port(QueryType, u32), + ScanTime(QueryType, u32), Version(QueryType, String), Protocol(QueryType, u32), MaxPlayers(QueryType, u32), @@ -690,19 +713,24 @@ pub fn search_parallel( ) -> Vec> { // Get column family handles let cf_addr = cfs[0]; - let cf_version = cfs[1]; - let cf_protocol = cfs[2]; - let cf_max_players = cfs[3]; - let cf_online_players = cfs[4]; - let cf_players_list = cfs[5]; - let cf_description = cfs[6]; - let cf_icon_hash = cfs[7]; - let cf_mod_info = cfs[8]; - let cf_forge_data = cfs[9]; - let cf_secure_chat = cfs[10]; - let cf_previews_chat = cfs[11]; + let cf_scan_time = cfs[1]; + let cf_version = cfs[2]; + let cf_protocol = cfs[3]; + let cf_max_players = cfs[4]; + let cf_online_players = cfs[5]; + let cf_players_list = cfs[6]; + let cf_description = cfs[7]; + let cf_icon_hash = cfs[8]; + let cf_mod_info = cfs[9]; + let cf_forge_data = cfs[10]; + let cf_secure_chat = cfs[11]; + let cf_previews_chat = cfs[12]; // Partition queries by type + let mut host_queries = Vec::new(); + let mut port_queries = Vec::new(); + + let mut time_queries = Vec::new(); let mut version_queries = Vec::new(); let mut protocol_queries = Vec::new(); let mut max_players_queries = Vec::new(); @@ -717,6 +745,9 @@ pub fn search_parallel( for q in queries { match q { + QueryDataType::Host(_, _) => host_queries.push(q), + QueryDataType::Port(_, _) => port_queries.push(q), + QueryDataType::ScanTime(_, _) => time_queries.push(q), QueryDataType::Version(_, _) => version_queries.push(q), QueryDataType::Protocol(_, _) => protocol_queries.push(q), QueryDataType::MaxPlayers(_, _) => max_players_queries.push(q), @@ -771,7 +802,14 @@ pub fn search_parallel( if let Some(bytes) = bytes { if let Ok(data) = std::str::from_utf8(&bytes) { queries.iter().all(|query| match query { - QueryDataType::Host(_, _) => false, + QueryDataType::Addr(_, _) => false, + QueryDataType::Host(qt, test) => { + match_string_comparison(qt, test, data.split(":").nth(0).unwrap_or("")) + } + QueryDataType::Port(qt, test) => { + match_num_comparison(qt, test, data.split(":").nth(1).unwrap_or("")) + } + QueryDataType::ScanTime(qt, test) => match_num_comparison(qt, test, data), QueryDataType::Version(qt, test) => match_string_comparison(qt, test, data), QueryDataType::Protocol(qt, test) => match_num_comparison(qt, test, data), QueryDataType::MaxPlayers(qt, test) => match_num_comparison(qt, test, data), @@ -814,7 +852,11 @@ pub fn search_parallel( .into_par_iter() .filter(|key| { // Check port queries - (version_queries.is_empty() || loop_queries(db, cf_version, key, &version_queries)) + (host_queries.is_empty() || loop_queries(db, cf_addr, key, &host_queries)) + && (port_queries.is_empty() || loop_queries(db, cf_addr, key, &port_queries)) + && (time_queries.is_empty() || loop_queries(db, cf_scan_time, key, &time_queries)) + && (version_queries.is_empty() + || loop_queries(db, cf_version, key, &version_queries)) && (protocol_queries.is_empty() || loop_queries(db, cf_protocol, key, &protocol_queries)) && (max_players_queries.is_empty() diff --git a/src/main.rs b/src/main.rs index cf98561..b82e475 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,10 +2,12 @@ use std::{ cmp::min, env, net::IpAddr, + str::FromStr, time::{Duration, Instant}, }; use parse_ip_range::parse_ip_targets; +use rand::{rng, seq::SliceRandom}; use untitled::{ database::ResultDatabase, online_scan, parse_ip_range, port_scan::tcp_scan, query, service_scan::service_scan::scan_services, @@ -35,6 +37,7 @@ fn main() -> Result<(), Box> { let _ = scan(database, args[2].to_string()); } } + "rescan" => rescan(database, args)?, // "search" => { // if args.len() != 4 { // println!("Invalid Usage!"); @@ -180,6 +183,56 @@ fn scan(database: ResultDatabase, arg: String) -> Result<(), Box) -> Result<(), Box> { + let start = Instant::now(); + if let Ok(query) = query::search(args[2..].join(" ")) { + let results = database.search(query); + if let Ok(results) = results { + let len = results.len(); + + let mut hosts: Vec = Vec::new(); + + for result in results { + println!("{}", result.to_string()); + hosts.push(IpAddr::from_str(result.ip.as_str()).unwrap()); + } + println!("{} results in {}ms", len, start.elapsed().as_millis()); + + hosts.sort(); + hosts.dedup(); + hosts.shuffle(&mut rng()); + + let chunks = hosts.chunks(BATCH_SIZE); + let num_chunks = chunks.len(); + for (i, hosts) in chunks.enumerate() { + let hosts = hosts.to_vec(); + let length = hosts.len(); + + println!("Scanning chunk {}/{} ({} hosts)", i + 1, num_chunks, length); + + let up_hosts: Vec = online_scan::ping_scanner::ping_scan(hosts).unwrap(); + let up_len = up_hosts.len(); + println!( + "Finished Pinging! {} Scanned, {} Up", + length, + up_hosts.len() + ); + + let tcp_results = + tcp_scan::tcp_scan(up_hosts, PORTS_1.to_vec(), Duration::from_secs(3)); + println!("Finished port scan"); + + let service_results = + scan_services(tcp_results, min(50, up_len), Duration::from_secs(1)); + println!("Finished service scan"); + let _ = database.add_data_row(service_results); + } + } + } + + Ok(()) +} + // fn search(database: ResultDatabase, search_type: String, arg: String) { // match search_type.as_str() { // "host" => { diff --git a/src/query.rs b/src/query.rs index 62788de..759eb5b 100644 --- a/src/query.rs +++ b/src/query.rs @@ -10,12 +10,12 @@ fn try_parse_host(query: &str) -> Option { let ip = IpAddr::from_str(split.nth(0).unwrap()); if let Some(port) = &split.nth(1) { if let (Ok(ip), Ok(port)) = (ip, port.parse::()) { - return Some(QueryDataType::Host(ip, port)); + return Some(QueryDataType::Addr(ip, port)); } } } if let Ok(ip) = IpAddr::from_str(&query) { - return Some(QueryDataType::Host(ip, 25565)); + return Some(QueryDataType::Addr(ip, 25565)); } None @@ -67,6 +67,20 @@ pub fn search(query: String) -> Result, Box { + results.push(QueryDataType::Port( + get_equals_type_num(&delim), + data.parse::().expect("Error parsing port"), + )); + Ok(()) + } + "scantime" => { + results.push(QueryDataType::ScanTime( + get_equals_type_num(&delim), + data.parse::().expect("Error parsing time"), + )); + Ok(()) + } "version" => { results.push(QueryDataType::Version(get_equals_type_str(&delim), data)); Ok(()) diff --git a/src/service_scan/tcp_minecraft.rs b/src/service_scan/tcp_minecraft.rs index 00c5a09..8e85799 100644 --- a/src/service_scan/tcp_minecraft.rs +++ b/src/service_scan/tcp_minecraft.rs @@ -3,10 +3,10 @@ use serde_json::json; use sha256::digest; use std::{ net::{IpAddr, SocketAddr, TcpStream}, - time::Duration, + time::{Duration, SystemTime, UNIX_EPOCH}, }; -use crate::database::DatabaseResult; +use crate::database::{DatabaseResult, EPOCH_2025}; pub fn scan( ip: IpAddr, @@ -30,6 +30,11 @@ pub fn scan( Ok(DatabaseResult { ip: ip.to_string(), port: port as u16, + time_scanned: (SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() + - EPOCH_2025 as u64) as u32, version: pong.version, protocol: pong.protocol as u32, max_players: pong.max_players as u32,