added read_folder (fixed #17)

Cargo.toml

@@ -11,6 +11,7 @@ repository = "https://github.com/qpxdesign/ngxav-rs/"
 [dependencies]
 chrono = "0.4.33"
 clap = { version = "4.4.18", features = ["derive"] }
+flate2 = "1.0.28"
 isbot = "0.1.3"
 lazy_static = "1.4.0"
 rand = "0.8.5"

README.md

@@ -14,7 +14,7 @@ To run this, use `ngxav`.
 If you're using the default NGINX Log Format, this should be working right out of the box.
 You can use the following flags to search using different parameters:
 
-- `-f/--file` - REQUIRED, specifcy the path to the file you wish to search through
+- `-f/--file` - REQUIRED, specifcy the path to the file/folder you wish to search through
 - `-s/--search`- OPTIONAL, either REGEX or text to match lines
 - `-b/--start_date`- OPTIONAL, find all logs within certain timespan (use time format 08/Nov/2023:08:04:05)
 - `-e/--end_date`- OPTIONAL, find all logs within certain timespan (use time format 08/Nov/2023:08:04:05)

src/main.rs

@@ -5,6 +5,7 @@ use std::collections::HashMap;
 use std::iter::FromIterator;
 use utils::keep_line::keep_line;
 use utils::parse_nginx_time_format::parse_nginx_time_format;
+use utils::read_folder::read_folder;
 use utils::sort_by_date;
 
 #[path = "./structs/mod.rs"]
@@ -54,7 +55,12 @@ fn main() {
         }
         return;
     }
-    let lines = lines_from_file(args.file).expect("should read");
+    let mut lines = Vec::new();
+    if args.file.contains("/") {
+        lines = utils::read_folder::read_folder(args.file);
+    } else {
+        lines = lines_from_file(args.file).expect("should read");
+    }
     let range = sort_by_date(&lines, &args.last, &args.start_date, &args.end_date);
     let mut kel: Vec<crate::structs::LineParseResult::LineParseResult> = lines[range.0..range.1]
         .par_iter()

src/utils/mod.rs

@@ -4,6 +4,7 @@ pub mod parse_input_time;
 pub mod parse_line;
 pub mod parse_nginx_time_format;
 pub mod parse_user_agent;
+pub mod read_folder;
 pub mod session_analytics;
 pub mod session_unique;
 pub mod sessionize;

src/utils/read_folder.rs

@@ -0,0 +1,26 @@
+extern crate flate2;
+use flate2::read::GzDecoder;
+use std::fs;
+use std::io;
+use std::io::prelude::*;
+
+pub fn read_folder(file_path: String) -> Vec<String> {
+    let paths = fs::read_dir(file_path).unwrap();
+    let mut lines = Vec::new();
+    for path in paths {
+        let a = std::io::BufReader::new(
+            fs::File::open(path.as_ref().unwrap().path().to_str().unwrap()).unwrap(),
+        );
+        if path.unwrap().path().to_str().unwrap().contains(".gz") {
+            let d = GzDecoder::new(a);
+            for line in io::BufReader::new(d).lines() {
+                lines.push(line.unwrap().to_string());
+            }
+        } else {
+            for line in io::BufReader::new(a).lines() {
+                lines.push(line.unwrap().to_string());
+            }
+        }
+    }
+    return lines;
+}

tests/logs_folder/100.log

@@ -0,0 +1,16 @@
+32774 - - [21/Jan/2024:18:08:00 -0500] "amtrak-api.marcmap.app" "GET /get-trains HTTP/1.1" 200 820909 "-" "AmTrack/15 CFNetwork/1490.0.4 Darwin/23.2.0" 0.381 {39.5296, -119.8142} <Reno, Nevada> -- (ATT-INTERNET4, 7018)
+538798 - - [21/Jan/2024:18:08:02 -0500] "amtrak-api.marcmap.app" "GET /get-trains HTTP/1.1" 200 820909 "-" "AmTrack/15 CFNetwork/1490.0.4 Darwin/23.2.0" 0.484 {37.5539, -77.4605} <Richmond, Virginia> -- (COMCAST-7922, 7922)
+273775 - - [21/Jan/2024:18:08:23 -0500] "amtrak-api.marcmap.app" "GET /get-trains HTTP/1.1" 200 820909 "-" "AmTrack/15 CFNetwork/1490.0.4 Darwin/23.2.0" 4.808 {40.3490, -74.6580} <Princeton, New Jersey> -- (ATT-MOBILITY-LLC-AS20057, 20057)
+273775 - - [21/Jan/2024:18:08:23 -0500] "amtrak-api.marcmap.app" "GET /get-trains HTTP/1.1" 200 820909 "-" "AmTrack/15 CFNetwork/1490.0.4 Darwin/23.2.0" 5.004 {40.3490, -74.6580} <Princeton, New Jersey> -- (ATT-MOBILITY-LLC-AS20057, 20057)
+32774 - - [21/Jan/2024:18:08:23 -0500] "amtrak-api.marcmap.app" "GET /get-trains HTTP/1.1" 200 820909 "-" "AmTrack/15 CFNetwork/1490.0.4 Darwin/23.2.0" 0.539 {39.5296, -119.8142} <Reno, Nevada> -- (ATT-INTERNET4, 7018)
+273775 - - [21/Jan/2024:18:08:41 -0500] "amtrak-api.marcmap.app" "GET /get-trains HTTP/1.1" 200 820909 "-" "AmTrack/15 CFNetwork/1490.0.4 Darwin/23.2.0" 1.749 {40.3490, -74.6580} <Princeton, New Jersey> -- (ATT-MOBILITY-LLC-AS20057, 20057)
+273775 - - [21/Jan/2024:18:08:41 -0500] "amtrak-api.marcmap.app" "GET /get-trains HTTP/1.1" 200 820909 "-" "AmTrack/15 CFNetwork/1490.0.4 Darwin/23.2.0" 2.108 {40.3490, -74.6580} <Princeton, New Jersey> -- (ATT-MOBILITY-LLC-AS20057, 20057)
+32774 - - [21/Jan/2024:18:08:41 -0500] "amtrak-api.marcmap.app" "GET /get-trains HTTP/1.1" 200 820909 "-" "AmTrack/15 CFNetwork/1490.0.4 Darwin/23.2.0" 0.790 {39.5296, -119.8142} <Reno, Nevada> -- (ATT-INTERNET4, 7018)
+273775 - - [21/Jan/2024:18:08:59 -0500] "amtrak-api.marcmap.app" "GET /get-trains HTTP/1.1" 200 820909 "-" "AmTrack/15 CFNetwork/1490.0.4 Darwin/23.2.0" 0.514 {40.3490, -74.6580} <Princeton, New Jersey> -- (ATT-MOBILITY-LLC-AS20057, 20057)
+273775 - - [21/Jan/2024:18:08:59 -0500] "amtrak-api.marcmap.app" "GET /get-trains HTTP/1.1" 200 820909 "-" "AmTrack/15 CFNetwork/1490.0.4 Darwin/23.2.0" 0.533 {40.3490, -74.6580} <Princeton, New Jersey> -- (ATT-MOBILITY-LLC-AS20057, 20057)
+32774 - - [21/Jan/2024:18:09:00 -0500] "amtrak-api.marcmap.app" "GET /get-trains HTTP/1.1" 200 820909 "-" "AmTrack/15 CFNetwork/1490.0.4 Darwin/23.2.0" 0.443 {39.5296, -119.8142} <Reno, Nevada> -- (ATT-INTERNET4, 7018)
+759244 - - [21/Jan/2024:18:09:13 -0500] "amtrak-api.marcmap.app" "GET /get-trains HTTP/1.1" 499 0 "-" "AmTrack/15 CFNetwork/1490.0.4 Darwin/23.2.0" 0.000 {38.9517, -92.3341} <Columbia, Missouri> -- (CLOUDFLARENET, 13335)
+759244 - - [21/Jan/2024:18:09:13 -0500] "amtrak-api.marcmap.app" "GET /get-trains HTTP/1.1" 200 792695 "-" "AmTrack/15 CFNetwork/1490.0.4 Darwin/23.2.0" 0.573 {38.9517, -92.3341} <Columbia, Missouri> -- (CLOUDFLARENET, 13335)
+32774 - - [21/Jan/2024:18:09:20 -0500] "amtrak-api.marcmap.app" "GET /get-trains HTTP/1.1" 200 792695 "-" "AmTrack/15 CFNetwork/1490.0.4 Darwin/23.2.0" 0.461 {39.5296, -119.8142} <Reno, Nevada> -- (ATT-INTERNET4, 7018)
+32774 - - [21/Jan/2024:18:09:26 -0500] "amtrak-api.marcmap.app" "GET /get-trains HTTP/1.1" 200 792695 "-" "AmTrack/15 CFNetwork/1490.0.4 Darwin/23.2.0" 0.445 {39.5296, -119.8142} <Reno, Nevada> -- (ATT-INTERNET4, 7018)
+32774 - - [21/Jan/2024:18:09:40 -0500] "amtrak-api.marcmap.app" "GET /get-trains HTTP/1.1" 200 792695 "-" "AmTrack/15 CFNetwork/1490.0.4 Darwin/23.2.0" 0.434 {39.5296, -119.8142} <Reno, Nevada> -- (ATT-INTERNET4, 7018)