From 3df9434a9044c2a2418fde217b0b249c10042dce Mon Sep 17 00:00:00 2001 From: Jacob Kiers Date: Sat, 6 Aug 2022 21:32:56 +0200 Subject: [PATCH] WIP: Message store Signed-off-by: Jacob Kiers --- .gitignore | 1 + Cargo.lock | 159 ++++++++++++++++++++++++++++++++++++++ bin/Cargo.toml | 2 + bin/src/main.rs | 15 +++- bin/src/message_reader.rs | 2 +- bin/src/storage.rs | 51 ++++++++++++ 6 files changed, 225 insertions(+), 5 deletions(-) create mode 100644 bin/src/storage.rs diff --git a/.gitignore b/.gitignore index a727c0a..d9bb7c3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /target /data +/output diff --git a/Cargo.lock b/Cargo.lock index 0b7d1d4..9244fa8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -48,6 +48,15 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -75,6 +84,12 @@ version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + [[package]] name = "cc" version = "1.0.73" @@ -110,6 +125,39 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "045ebe27666471bb549370b4b0b3e51b07f56325befa4284db65fc89c02511b1" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "once_cell", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -220,6 +268,25 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.14.6" @@ -260,6 +327,15 @@ dependencies = [ "nom", ] +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + [[package]] name = "js-sys" version = "0.3.59" @@ -294,6 +370,16 @@ version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "505e71a4706fa491e9b1b55f51b95d4037d0821ee40131190475f692b35b009b" +[[package]] +name = "lock_api" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.17" @@ -319,6 +405,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + [[package]] name = "never" version = "0.1.0" @@ -331,11 +426,13 @@ version = "0.1.0" dependencies = [ "atom_syndication", "base16ct", + "bincode", "chrono", "imap", "mail-parser", "rustls-connector", "sha2", + "sled", ] [[package]] @@ -374,6 +471,31 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + [[package]] name = "proc-macro2" version = "1.0.43" @@ -402,6 +524,15 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.6.0" @@ -464,6 +595,12 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + [[package]] name = "sct" version = "0.7.0" @@ -505,6 +642,28 @@ dependencies = [ "digest", ] +[[package]] +name = "sled" +version = "0.34.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" +dependencies = [ + "crc32fast", + "crossbeam-epoch", + "crossbeam-utils", + "fs2", + "fxhash", + "libc", + "log", + "parking_lot", +] + +[[package]] +name = "smallvec" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" + [[package]] name = "spin" version = "0.5.2" diff --git a/bin/Cargo.toml b/bin/Cargo.toml index a824381..bc53599 100644 --- a/bin/Cargo.toml +++ b/bin/Cargo.toml @@ -9,8 +9,10 @@ description = "Converts email newsletters to static HTML files" [dependencies] atom_syndication = "^0.11.0" base16ct = { version = "^0.1.0", features = [ "alloc" ] } +bincode = "^1.3.3" chrono = "^0.4" imap = { version = "^2.4.1", default-features = false } mail-parser = "^0.5.0" rustls-connector = { version = "^0.16.1", default-features = false, features = [ "webpki-roots-certs", "quic" ] } sha2 = "^0.10.2" +sled = "^0.34.7" \ No newline at end of file diff --git a/bin/src/main.rs b/bin/src/main.rs index 2bd6f4e..a800c0d 100644 --- a/bin/src/main.rs +++ b/bin/src/main.rs @@ -1,20 +1,23 @@ mod message_reader; +mod storage; use std::{ fs::{File, OpenOptions}, io::Write, - path::{Path, PathBuf}, + path::{Path, PathBuf}, error::Error, }; use atom_syndication::{ ContentBuilder, Entry, EntryBuilder, Feed, FeedBuilder, Generator, LinkBuilder, Person, }; + use chrono::{DateTime, TimeZone, Utc}; use mail_parser::{HeaderValue, Message as MpMessage}; use sha2::{Digest, Sha256}; use message_reader::{EmailReader, TestMessagesReader}; +use storage::Store; pub struct Message { uid: String, @@ -35,8 +38,8 @@ impl Message { } } -fn main() { - let dir = Path::new("data"); +fn main() -> Result<(), Box> { + let dir = Path::new("output"); if !dir.exists() { std::fs::create_dir(&dir).expect("Could not create directory"); } @@ -44,9 +47,11 @@ fn main() { let mut feed = build_atom_feed(); let mut reader = TestMessagesReader::new((&Path::new("tests/data")).to_path_buf()); + let store = Store::load_database_for_mailbox("newsletters@kie.rs")?; for msg in reader.read_rfc822_messages() { println!("Processing message {}", msg.get_uid()); + store.store_mail(&msg)?; let parsed = msg.get_parsed().expect("A parsed messsage."); @@ -75,6 +80,8 @@ fn main() { feed.set_updated(Utc::now()); let _ = feed.write_to(File::create(format!("{}/feed.atom", dir.display())).unwrap()); } + + Ok(()) } fn add_entry_to_feed(feed: &mut Feed, message: &Message, processed_html: &String) { @@ -165,7 +172,7 @@ fn build_atom_feed() -> Feed { .build() } -fn write_to_test_path(msg: &Message) { +fn _write_to_test_path(msg: &Message) { let test_path: PathBuf = [ Path::new("tests/data"), Path::new(&format!("{}.eml", &msg.get_uid())), diff --git a/bin/src/message_reader.rs b/bin/src/message_reader.rs index b24d245..714bf4c 100644 --- a/bin/src/message_reader.rs +++ b/bin/src/message_reader.rs @@ -66,7 +66,7 @@ pub struct ImapReader { } impl ImapReader { - pub fn new(host: String, port: u16, username: String, password: String) -> Self { + pub fn _new(host: String, port: u16, username: String, password: String) -> Self { ImapReader { host, port, diff --git a/bin/src/storage.rs b/bin/src/storage.rs new file mode 100644 index 0000000..ff75894 --- /dev/null +++ b/bin/src/storage.rs @@ -0,0 +1,51 @@ +use sled::{Db, Transactional}; + +use crate::Message; +use std::str::FromStr; + +pub(crate) struct Store { + db: Db, + mailbox: String, +} + +type ER = Result<(), sled::Error>; +type BR = Result; + +impl Store { + pub fn load_database_for_mailbox>(mailbox: S) -> Result { + let db = sled::open("data/maildb")?; + Ok(Store { + db, + mailbox: mailbox.into(), + }) + } + + pub fn store_mail(&self, message: &Message) -> ER { + self.mb()?.insert(message.get_uid(), &*message.data)?; + Ok(()) + } + + pub fn has_mail>(&self, uid: S) -> BR { + self.mb()?.contains_key(uid.into()) + } + + pub fn mark_in_feed>(&self, uid: S) -> ER { + self.feed()?.insert(uid.into(), &[1])?; + Ok(()) + } + + pub fn is_in_feed>(&self, uid: S) -> BR { + match self.feed()?.get(uid.into())? { + Some(v) => Ok(bincode::deserialize(&v).expect("Cannot convert to bool")), + None => Ok(false), + } + } + + fn mb(&self) -> Result { + self.db.open_tree(&self.mailbox) + } + + fn feed(&self) -> Result { + self.db.open_tree("feed") + } +}