From cd000061b459c6eac2bdba638ff3701f2f9f5e06 Mon Sep 17 00:00:00 2001 From: Jacob Kiers Date: Fri, 21 Nov 2025 17:04:31 +0100 Subject: [PATCH] Implemented debug logging to debug_logs/ --- .gitignore | 1 + Cargo.lock | 49 ++++++++++++++ Cargo.toml | 7 +- banks2ff/Cargo.toml | 8 +++ banks2ff/src/debug.rs | 116 ++++++++++++++++++++++++++++++++ banks2ff/src/main.rs | 28 +++++++- firefly-client/Cargo.toml | 3 +- firefly-client/src/client.rs | 13 +++- gocardless-client/Cargo.toml | 3 +- gocardless-client/src/client.rs | 13 +++- 10 files changed, 228 insertions(+), 13 deletions(-) create mode 100644 banks2ff/src/debug.rs diff --git a/.gitignore b/.gitignore index 0120d95..df3b34e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ **/target/ **/*.rs.bk .env +/debug_logs/ diff --git a/Cargo.lock b/Cargo.lock index cd55cef..116f7e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -159,15 +159,21 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "bytes", "chrono", "clap", "dotenvy", "firefly-client", "gocardless-client", + "http", + "hyper", "mockall", + "reqwest", + "reqwest-middleware", "rust_decimal", "serde", "serde_json", + "task-local-extensions", "tokio", "tracing", "tracing-subscriber", @@ -475,6 +481,7 @@ version = "0.1.0" dependencies = [ "chrono", "reqwest", + "reqwest-middleware", "rust_decimal", "serde", "serde_json", @@ -660,6 +667,7 @@ version = "0.1.0" dependencies = [ "chrono", "reqwest", + "reqwest-middleware", "serde", "serde_json", "thiserror", @@ -1051,6 +1059,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "mio" version = "1.1.0" @@ -1421,6 +1439,7 @@ dependencies = [ "js-sys", "log", "mime", + "mime_guess", "once_cell", "percent-encoding", "pin-project-lite", @@ -1442,6 +1461,21 @@ dependencies = [ "winreg", ] +[[package]] +name = "reqwest-middleware" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a735987236a8e238bf0296c7e351b999c188ccc11477f311b82b55c93984216" +dependencies = [ + "anyhow", + "async-trait", + "http", + "reqwest", + "serde", + "task-local-extensions", + "thiserror", +] + [[package]] name = "retain_mut" version = "0.1.9" @@ -1778,6 +1812,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "task-local-extensions" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba323866e5d033818e3240feeb9f7db2c4296674e4d9e16b97b7bf8f490434e8" +dependencies = [ + "pin-utils", +] + [[package]] name = "termtree" version = "0.5.1" @@ -2016,6 +2059,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "unicase" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" + [[package]] name = "unicode-ident" version = "1.0.22" diff --git a/Cargo.toml b/Cargo.toml index a683af1..3630c75 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,8 +24,11 @@ rust_decimal = { version = "1.33", features = ["serde-float"] } async-trait = "0.1" dotenvy = "0.15" clap = { version = "4.4", features = ["derive", "env"] } -reqwest = { version = "0.11", features = ["json", "multipart"] } +reqwest = { version = "0.11", default-features = false, features = ["json", "multipart", "rustls-tls"] } url = "2.5" wiremock = "0.5" tokio-test = "0.4" -mockall = "0.11" + mockall = "0.11" +reqwest-middleware = "0.2" +hyper = { version = "0.14", features = ["full"] } +bytes = "1.0" diff --git a/banks2ff/Cargo.toml b/banks2ff/Cargo.toml index 2ee8377..6ff0529 100644 --- a/banks2ff/Cargo.toml +++ b/banks2ff/Cargo.toml @@ -15,6 +15,7 @@ chrono = { workspace = true } rust_decimal = { workspace = true } dotenvy = { workspace = true } clap = { workspace = true } +reqwest = { workspace = true } # Core logic dependencies async-trait = { workspace = true } @@ -23,5 +24,12 @@ async-trait = { workspace = true } firefly-client = { path = "../firefly-client" } gocardless-client = { path = "../gocardless-client" } +# Debug logging dependencies +reqwest-middleware = { workspace = true } +hyper = { workspace = true } +bytes = { workspace = true } +http = "0.2" +task-local-extensions = "0.1" + [dev-dependencies] mockall = { workspace = true } diff --git a/banks2ff/src/debug.rs b/banks2ff/src/debug.rs new file mode 100644 index 0000000..7808966 --- /dev/null +++ b/banks2ff/src/debug.rs @@ -0,0 +1,116 @@ +use reqwest_middleware::{Middleware, Next}; +use task_local_extensions::Extensions; +use reqwest::{Request, Response}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::fs; +use std::path::Path; +use chrono::Utc; +use hyper::Body; + +static REQUEST_COUNTER: AtomicU64 = AtomicU64::new(0); + +pub struct DebugLogger { + service_name: String, +} + +impl DebugLogger { + pub fn new(service_name: &str) -> Self { + Self { + service_name: service_name.to_string(), + } + } +} + +#[async_trait::async_trait] +impl Middleware for DebugLogger { + async fn handle( + &self, + req: Request, + extensions: &mut Extensions, + next: Next<'_>, + ) -> reqwest_middleware::Result { + let request_id = REQUEST_COUNTER.fetch_add(1, Ordering::SeqCst); + let timestamp = Utc::now().format("%Y%m%d_%H%M%S"); + let filename = format!("{}_{}_{}.txt", timestamp, request_id, self.service_name); + + let dir = format!("./debug_logs/{}", self.service_name); + fs::create_dir_all(&dir).unwrap_or_else(|e| { + eprintln!("Failed to create debug log directory: {}", e); + }); + + let filepath = Path::new(&dir).join(filename); + + let mut log_content = String::new(); + + // Curl command + log_content.push_str("# Curl command:\n"); + let curl = build_curl_command(&req); + log_content.push_str(&format!("{}\n\n", curl)); + + // Request + log_content.push_str("# Request:\n"); + log_content.push_str(&format!("{} {} HTTP/1.1\n", req.method(), req.url())); + for (key, value) in req.headers() { + log_content.push_str(&format!("{}: {}\n", key, value.to_str().unwrap_or("[INVALID]"))); + } + if let Some(body) = req.body() { + if let Some(bytes) = body.as_bytes() { + log_content.push_str(&format!("\n{}", String::from_utf8_lossy(bytes))); + } + } + log_content.push_str("\n\n"); + + // Send request and get response + let response = next.run(req, extensions).await?; + + // Extract parts before consuming body + let status = response.status(); + let version = response.version(); + let headers = response.headers().clone(); + + // Response + log_content.push_str("# Response:\n"); + log_content.push_str(&format!("HTTP/1.1 {} {}\n", status.as_u16(), status.canonical_reason().unwrap_or("Unknown"))); + for (key, value) in &headers { + log_content.push_str(&format!("{}: {}\n", key, value.to_str().unwrap_or("[INVALID]"))); + } + + // Read body + let body_bytes = response.bytes().await.map_err(|e| reqwest_middleware::Error::Middleware(anyhow::anyhow!("Failed to read response body: {}", e)))?; + let body_str = String::from_utf8_lossy(&body_bytes); + log_content.push_str(&format!("\n{}", body_str)); + + // Write to file + if let Err(e) = fs::write(&filepath, log_content) { + eprintln!("Failed to write debug log: {}", e); + } + + // Reconstruct response + let mut builder = http::Response::builder() + .status(status) + .version(version); + for (key, value) in &headers { + builder = builder.header(key, value); + } + let new_response = builder.body(Body::from(body_bytes)).unwrap(); + Ok(Response::from(new_response)) + } +} + +fn build_curl_command(req: &Request) -> String { + let mut curl = format!("curl -v -X {} '{}'", req.method(), req.url()); + + for (key, value) in req.headers() { + let value_str = value.to_str().unwrap_or("[INVALID]").replace("'", "\\'"); + curl.push_str(&format!(" -H '{}: {}'", key, value_str)); + } + + if let Some(body) = req.body() { + if let Some(bytes) = body.as_bytes() { + let body_str = String::from_utf8_lossy(bytes).replace("'", "\\'"); + curl.push_str(&format!(" -d '{}'", body_str)); + } + } + + curl +} \ No newline at end of file diff --git a/banks2ff/src/main.rs b/banks2ff/src/main.rs index 440645c..65c1d50 100644 --- a/banks2ff/src/main.rs +++ b/banks2ff/src/main.rs @@ -1,13 +1,16 @@ mod adapters; mod core; +mod debug; use clap::Parser; use tracing::{info, error}; use crate::adapters::gocardless::client::GoCardlessAdapter; use crate::adapters::firefly::client::FireflyAdapter; use crate::core::sync::run_sync; +use crate::debug::DebugLogger; use gocardless_client::client::GoCardlessClient; use firefly_client::client::FireflyClient; +use reqwest_middleware::ClientBuilder; use std::env; use chrono::NaiveDate; @@ -29,6 +32,10 @@ struct Args { /// Dry run mode: Do not create or update transactions in Firefly III. #[arg(long, default_value_t = false)] dry_run: bool, + + /// Enable debug logging of HTTP requests/responses to ./debug_logs/ + #[arg(long, default_value_t = false)] + debug: bool, } #[tokio::main] @@ -52,13 +59,28 @@ async fn main() -> anyhow::Result<()> { let gc_url = env::var("GOCARDLESS_URL").unwrap_or_else(|_| "https://bankaccountdata.gocardless.com".to_string()); let gc_id = env::var("GOCARDLESS_ID").expect("GOCARDLESS_ID not set"); let gc_key = env::var("GOCARDLESS_KEY").expect("GOCARDLESS_KEY not set"); - + let ff_url = env::var("FIREFLY_III_URL").expect("FIREFLY_III_URL not set"); let ff_key = env::var("FIREFLY_III_API_KEY").expect("FIREFLY_III_API_KEY not set"); // Clients - let gc_client = GoCardlessClient::new(&gc_url, &gc_id, &gc_key)?; - let ff_client = FireflyClient::new(&ff_url, &ff_key)?; + let gc_client = if args.debug { + let client = ClientBuilder::new(reqwest::Client::new()) + .with(DebugLogger::new("gocardless")) + .build(); + GoCardlessClient::with_client(&gc_url, &gc_id, &gc_key, Some(client))? + } else { + GoCardlessClient::new(&gc_url, &gc_id, &gc_key)? + }; + + let ff_client = if args.debug { + let client = ClientBuilder::new(reqwest::Client::new()) + .with(DebugLogger::new("firefly")) + .build(); + FireflyClient::with_client(&ff_url, &ff_key, Some(client))? + } else { + FireflyClient::new(&ff_url, &ff_key)? + }; // Adapters let source = GoCardlessAdapter::new(gc_client); diff --git a/firefly-client/Cargo.toml b/firefly-client/Cargo.toml index abc0ee1..d184c3a 100644 --- a/firefly-client/Cargo.toml +++ b/firefly-client/Cargo.toml @@ -5,7 +5,8 @@ edition.workspace = true authors.workspace = true [dependencies] -reqwest = { version = "0.11", default-features = false, features = ["json", "rustls-tls"] } +reqwest = { workspace = true, default-features = false, features = ["json", "rustls-tls"] } +reqwest-middleware = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } diff --git a/firefly-client/src/client.rs b/firefly-client/src/client.rs index 0e343a4..f0dd43d 100644 --- a/firefly-client/src/client.rs +++ b/firefly-client/src/client.rs @@ -1,4 +1,5 @@ -use reqwest::{Client, Url}; +use reqwest::Url; +use reqwest_middleware::ClientWithMiddleware; use serde::de::DeserializeOwned; use thiserror::Error; use tracing::instrument; @@ -8,6 +9,8 @@ use crate::models::{AccountArray, TransactionStore, TransactionArray, Transactio pub enum FireflyError { #[error("Request failed: {0}")] RequestFailed(#[from] reqwest::Error), + #[error("Middleware error: {0}")] + MiddlewareError(#[from] reqwest_middleware::Error), #[error("API Error: {0}")] ApiError(String), #[error("URL Parse Error: {0}")] @@ -16,15 +19,19 @@ pub enum FireflyError { pub struct FireflyClient { base_url: Url, - client: Client, + client: ClientWithMiddleware, access_token: String, } impl FireflyClient { pub fn new(base_url: &str, access_token: &str) -> Result { + Self::with_client(base_url, access_token, None) + } + + pub fn with_client(base_url: &str, access_token: &str, client: Option) -> Result { Ok(Self { base_url: Url::parse(base_url)?, - client: Client::new(), + client: client.unwrap_or_else(|| reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build()), access_token: access_token.to_string(), }) } diff --git a/gocardless-client/Cargo.toml b/gocardless-client/Cargo.toml index 3bdf8a7..3140e8f 100644 --- a/gocardless-client/Cargo.toml +++ b/gocardless-client/Cargo.toml @@ -5,7 +5,8 @@ edition.workspace = true authors.workspace = true [dependencies] -reqwest = { version = "0.11", default-features = false, features = ["json", "rustls-tls"] } +reqwest = { workspace = true, default-features = false, features = ["json", "rustls-tls"] } +reqwest-middleware = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } diff --git a/gocardless-client/src/client.rs b/gocardless-client/src/client.rs index 1497265..55a6cb8 100644 --- a/gocardless-client/src/client.rs +++ b/gocardless-client/src/client.rs @@ -1,4 +1,5 @@ -use reqwest::{Client, Url}; +use reqwest::Url; +use reqwest_middleware::ClientWithMiddleware; use serde::{Deserialize, Serialize}; use thiserror::Error; use tracing::{debug, instrument}; @@ -8,6 +9,8 @@ use crate::models::{TokenResponse, PaginatedResponse, Requisition, Account, Tran pub enum GoCardlessError { #[error("Request failed: {0}")] RequestFailed(#[from] reqwest::Error), + #[error("Middleware error: {0}")] + MiddlewareError(#[from] reqwest_middleware::Error), #[error("API Error: {0}")] ApiError(String), #[error("Serialization error: {0}")] @@ -18,7 +21,7 @@ pub enum GoCardlessError { pub struct GoCardlessClient { base_url: Url, - client: Client, + client: ClientWithMiddleware, secret_id: String, secret_key: String, access_token: Option, @@ -33,9 +36,13 @@ struct TokenRequest<'a> { impl GoCardlessClient { pub fn new(base_url: &str, secret_id: &str, secret_key: &str) -> Result { + Self::with_client(base_url, secret_id, secret_key, None) + } + + pub fn with_client(base_url: &str, secret_id: &str, secret_key: &str, client: Option) -> Result { Ok(Self { base_url: Url::parse(base_url)?, - client: Client::new(), + client: client.unwrap_or_else(|| reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build()), secret_id: secret_id.to_string(), secret_key: secret_key.to_string(), access_token: None,