Implemented debug logging to debug_logs/

.gitignore

@@ -2,3 +2,4 @@
 **/target/
 **/*.rs.bk
 .env
+/debug_logs/

Cargo.lock

@@ -159,15 +159,21 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "async-trait",
+ "bytes",
  "chrono",
  "clap",
  "dotenvy",
  "firefly-client",
  "gocardless-client",
+ "http",
+ "hyper",
  "mockall",
+ "reqwest",
+ "reqwest-middleware",
  "rust_decimal",
  "serde",
  "serde_json",
+ "task-local-extensions",
  "tokio",
  "tracing",
  "tracing-subscriber",
@@ -475,6 +481,7 @@ version = "0.1.0"
 dependencies = [
  "chrono",
  "reqwest",
+ "reqwest-middleware",
  "rust_decimal",
  "serde",
  "serde_json",
@@ -660,6 +667,7 @@ version = "0.1.0"
 dependencies = [
  "chrono",
  "reqwest",
+ "reqwest-middleware",
  "serde",
  "serde_json",
  "thiserror",
@@ -1051,6 +1059,16 @@ version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
 
+[[package]]
+name = "mime_guess"
+version = "2.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
+dependencies = [
+ "mime",
+ "unicase",
+]
+
 [[package]]
 name = "mio"
 version = "1.1.0"
@@ -1421,6 +1439,7 @@ dependencies = [
  "js-sys",
  "log",
  "mime",
+ "mime_guess",
  "once_cell",
  "percent-encoding",
  "pin-project-lite",
@@ -1442,6 +1461,21 @@ dependencies = [
  "winreg",
 ]
 
+[[package]]
+name = "reqwest-middleware"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a735987236a8e238bf0296c7e351b999c188ccc11477f311b82b55c93984216"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "http",
+ "reqwest",
+ "serde",
+ "task-local-extensions",
+ "thiserror",
+]
+
 [[package]]
 name = "retain_mut"
 version = "0.1.9"
@@ -1778,6 +1812,15 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
 
+[[package]]
+name = "task-local-extensions"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba323866e5d033818e3240feeb9f7db2c4296674e4d9e16b97b7bf8f490434e8"
+dependencies = [
+ "pin-utils",
+]
+
 [[package]]
 name = "termtree"
 version = "0.5.1"
@@ -2016,6 +2059,12 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
 
+[[package]]
+name = "unicase"
+version = "2.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.22"

Cargo.toml

@@ -24,8 +24,11 @@ rust_decimal = { version = "1.33", features = ["serde-float"] }
 async-trait = "0.1"
 dotenvy = "0.15"
 clap = { version = "4.4", features = ["derive", "env"] }
-reqwest = { version = "0.11", features = ["json", "multipart"] }
+reqwest = { version = "0.11", default-features = false, features = ["json", "multipart", "rustls-tls"] }
 url = "2.5"
 wiremock = "0.5"
 tokio-test = "0.4"
-mockall = "0.11"
+ mockall = "0.11"
+reqwest-middleware = "0.2"
+hyper = { version = "0.14", features = ["full"] }
+bytes = "1.0"

banks2ff/Cargo.toml

@@ -15,6 +15,7 @@ chrono = { workspace = true }
 rust_decimal = { workspace = true }
 dotenvy = { workspace = true }
 clap = { workspace = true }
+reqwest = { workspace = true }
 
 # Core logic dependencies
 async-trait = { workspace = true }
@@ -23,5 +24,12 @@ async-trait = { workspace = true }
 firefly-client = { path = "../firefly-client" }
 gocardless-client = { path = "../gocardless-client" }
 
+# Debug logging dependencies
+reqwest-middleware = { workspace = true }
+hyper = { workspace = true }
+bytes = { workspace = true }
+http = "0.2"
+task-local-extensions = "0.1"
+
 [dev-dependencies]
 mockall = { workspace = true }

banks2ff/src/debug.rs

@@ -0,0 +1,116 @@
+use reqwest_middleware::{Middleware, Next};
+use task_local_extensions::Extensions;
+use reqwest::{Request, Response};
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::fs;
+use std::path::Path;
+use chrono::Utc;
+use hyper::Body;
+
+static REQUEST_COUNTER: AtomicU64 = AtomicU64::new(0);
+
+pub struct DebugLogger {
+    service_name: String,
+}
+
+impl DebugLogger {
+    pub fn new(service_name: &str) -> Self {
+        Self {
+            service_name: service_name.to_string(),
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl Middleware for DebugLogger {
+    async fn handle(
+        &self,
+        req: Request,
+        extensions: &mut Extensions,
+        next: Next<'_>,
+    ) -> reqwest_middleware::Result<Response> {
+        let request_id = REQUEST_COUNTER.fetch_add(1, Ordering::SeqCst);
+        let timestamp = Utc::now().format("%Y%m%d_%H%M%S");
+        let filename = format!("{}_{}_{}.txt", timestamp, request_id, self.service_name);
+
+        let dir = format!("./debug_logs/{}", self.service_name);
+        fs::create_dir_all(&dir).unwrap_or_else(|e| {
+            eprintln!("Failed to create debug log directory: {}", e);
+        });
+
+        let filepath = Path::new(&dir).join(filename);
+
+        let mut log_content = String::new();
+
+        // Curl command
+        log_content.push_str("# Curl command:\n");
+        let curl = build_curl_command(&req);
+        log_content.push_str(&format!("{}\n\n", curl));
+
+        // Request
+        log_content.push_str("# Request:\n");
+        log_content.push_str(&format!("{} {} HTTP/1.1\n", req.method(), req.url()));
+        for (key, value) in req.headers() {
+            log_content.push_str(&format!("{}: {}\n", key, value.to_str().unwrap_or("[INVALID]")));
+        }
+        if let Some(body) = req.body() {
+            if let Some(bytes) = body.as_bytes() {
+                log_content.push_str(&format!("\n{}", String::from_utf8_lossy(bytes)));
+            }
+        }
+        log_content.push_str("\n\n");
+
+        // Send request and get response
+        let response = next.run(req, extensions).await?;
+
+        // Extract parts before consuming body
+        let status = response.status();
+        let version = response.version();
+        let headers = response.headers().clone();
+
+        // Response
+        log_content.push_str("# Response:\n");
+        log_content.push_str(&format!("HTTP/1.1 {} {}\n", status.as_u16(), status.canonical_reason().unwrap_or("Unknown")));
+        for (key, value) in &headers {
+            log_content.push_str(&format!("{}: {}\n", key, value.to_str().unwrap_or("[INVALID]")));
+        }
+
+        // Read body
+        let body_bytes = response.bytes().await.map_err(|e| reqwest_middleware::Error::Middleware(anyhow::anyhow!("Failed to read response body: {}", e)))?;
+        let body_str = String::from_utf8_lossy(&body_bytes);
+        log_content.push_str(&format!("\n{}", body_str));
+
+        // Write to file
+        if let Err(e) = fs::write(&filepath, log_content) {
+            eprintln!("Failed to write debug log: {}", e);
+        }
+
+        // Reconstruct response
+        let mut builder = http::Response::builder()
+            .status(status)
+            .version(version);
+        for (key, value) in &headers {
+            builder = builder.header(key, value);
+        }
+        let new_response = builder.body(Body::from(body_bytes)).unwrap();
+        Ok(Response::from(new_response))
+    }
+}
+
+fn build_curl_command(req: &Request) -> String {
+    let mut curl = format!("curl -v -X {} '{}'", req.method(), req.url());
+
+    for (key, value) in req.headers() {
+        let value_str = value.to_str().unwrap_or("[INVALID]").replace("'", "\\'");
+        curl.push_str(&format!(" -H '{}: {}'", key, value_str));
+    }
+
+    if let Some(body) = req.body() {
+        if let Some(bytes) = body.as_bytes() {
+            let body_str = String::from_utf8_lossy(bytes).replace("'", "\\'");
+            curl.push_str(&format!(" -d '{}'", body_str));
+        }
+    }
+
+    curl
+}

banks2ff/src/main.rs

@@ -1,13 +1,16 @@
 mod adapters;
 mod core;
+mod debug;
 
 use clap::Parser;
 use tracing::{info, error};
 use crate::adapters::gocardless::client::GoCardlessAdapter;
 use crate::adapters::firefly::client::FireflyAdapter;
 use crate::core::sync::run_sync;
+use crate::debug::DebugLogger;
 use gocardless_client::client::GoCardlessClient;
 use firefly_client::client::FireflyClient;
+use reqwest_middleware::ClientBuilder;
 use std::env;
 use chrono::NaiveDate;
 
@@ -29,6 +32,10 @@ struct Args {
     /// Dry run mode: Do not create or update transactions in Firefly III.
     #[arg(long, default_value_t = false)]
     dry_run: bool,
+
+    /// Enable debug logging of HTTP requests/responses to ./debug_logs/
+    #[arg(long, default_value_t = false)]
+    debug: bool,
 }
 
 #[tokio::main]
@@ -52,13 +59,28 @@ async fn main() -> anyhow::Result<()> {
     let gc_url = env::var("GOCARDLESS_URL").unwrap_or_else(|_| "https://bankaccountdata.gocardless.com".to_string());
     let gc_id = env::var("GOCARDLESS_ID").expect("GOCARDLESS_ID not set");
     let gc_key = env::var("GOCARDLESS_KEY").expect("GOCARDLESS_KEY not set");
-    
+
     let ff_url = env::var("FIREFLY_III_URL").expect("FIREFLY_III_URL not set");
     let ff_key = env::var("FIREFLY_III_API_KEY").expect("FIREFLY_III_API_KEY not set");
 
     // Clients
-    let gc_client = GoCardlessClient::new(&gc_url, &gc_id, &gc_key)?;
+    let gc_client = if args.debug {
-    let ff_client = FireflyClient::new(&ff_url, &ff_key)?;
+        let client = ClientBuilder::new(reqwest::Client::new())
+            .with(DebugLogger::new("gocardless"))
+            .build();
+        GoCardlessClient::with_client(&gc_url, &gc_id, &gc_key, Some(client))?
+    } else {
+        GoCardlessClient::new(&gc_url, &gc_id, &gc_key)?
+    };
+
+    let ff_client = if args.debug {
+        let client = ClientBuilder::new(reqwest::Client::new())
+            .with(DebugLogger::new("firefly"))
+            .build();
+        FireflyClient::with_client(&ff_url, &ff_key, Some(client))?
+    } else {
+        FireflyClient::new(&ff_url, &ff_key)?
+    };
 
     // Adapters
     let source = GoCardlessAdapter::new(gc_client);

firefly-client/Cargo.toml

@@ -5,7 +5,8 @@ edition.workspace = true
 authors.workspace = true
 
 [dependencies]
-reqwest = { version = "0.11", default-features = false, features = ["json", "rustls-tls"] }
+reqwest = { workspace = true, default-features = false, features = ["json", "rustls-tls"] }
+reqwest-middleware = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 thiserror = { workspace = true }

firefly-client/src/client.rs

@@ -1,4 +1,5 @@
-use reqwest::{Client, Url};
+use reqwest::Url;
+use reqwest_middleware::ClientWithMiddleware;
 use serde::de::DeserializeOwned;
 use thiserror::Error;
 use tracing::instrument;
@@ -8,6 +9,8 @@ use crate::models::{AccountArray, TransactionStore, TransactionArray, Transactio
 pub enum FireflyError {
     #[error("Request failed: {0}")]
     RequestFailed(#[from] reqwest::Error),
+    #[error("Middleware error: {0}")]
+    MiddlewareError(#[from] reqwest_middleware::Error),
     #[error("API Error: {0}")]
     ApiError(String),
     #[error("URL Parse Error: {0}")]
@@ -16,15 +19,19 @@ pub enum FireflyError {
 
 pub struct FireflyClient {
     base_url: Url,
-    client: Client,
+    client: ClientWithMiddleware,
     access_token: String,
 }
 
 impl FireflyClient {
     pub fn new(base_url: &str, access_token: &str) -> Result<Self, FireflyError> {
+        Self::with_client(base_url, access_token, None)
+    }
+
+    pub fn with_client(base_url: &str, access_token: &str, client: Option<ClientWithMiddleware>) -> Result<Self, FireflyError> {
         Ok(Self {
             base_url: Url::parse(base_url)?,
-            client: Client::new(),
+            client: client.unwrap_or_else(|| reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build()),
             access_token: access_token.to_string(),
         })
     }

gocardless-client/Cargo.toml

@@ -5,7 +5,8 @@ edition.workspace = true
 authors.workspace = true
 
 [dependencies]
-reqwest = { version = "0.11", default-features = false, features = ["json", "rustls-tls"] }
+reqwest = { workspace = true, default-features = false, features = ["json", "rustls-tls"] }
+reqwest-middleware = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 thiserror = { workspace = true }

gocardless-client/src/client.rs

@@ -1,4 +1,5 @@
-use reqwest::{Client, Url};
+use reqwest::Url;
+use reqwest_middleware::ClientWithMiddleware;
 use serde::{Deserialize, Serialize};
 use thiserror::Error;
 use tracing::{debug, instrument};
@@ -8,6 +9,8 @@ use crate::models::{TokenResponse, PaginatedResponse, Requisition, Account, Tran
 pub enum GoCardlessError {
     #[error("Request failed: {0}")]
     RequestFailed(#[from] reqwest::Error),
+    #[error("Middleware error: {0}")]
+    MiddlewareError(#[from] reqwest_middleware::Error),
     #[error("API Error: {0}")]
     ApiError(String),
     #[error("Serialization error: {0}")]
@@ -18,7 +21,7 @@ pub enum GoCardlessError {
 
 pub struct GoCardlessClient {
     base_url: Url,
-    client: Client,
+    client: ClientWithMiddleware,
     secret_id: String,
     secret_key: String,
     access_token: Option<String>,
@@ -33,9 +36,13 @@ struct TokenRequest<'a> {
 
 impl GoCardlessClient {
     pub fn new(base_url: &str, secret_id: &str, secret_key: &str) -> Result<Self, GoCardlessError> {
+        Self::with_client(base_url, secret_id, secret_key, None)
+    }
+
+    pub fn with_client(base_url: &str, secret_id: &str, secret_key: &str, client: Option<ClientWithMiddleware>) -> Result<Self, GoCardlessError> {
         Ok(Self {
             base_url: Url::parse(base_url)?,
-            client: Client::new(),
+            client: client.unwrap_or_else(|| reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build()),
             secret_id: secret_id.to_string(),
             secret_key: secret_key.to_string(),
             access_token: None,