Compare commits

...

1 Commits

Author SHA1 Message Date
Jacob Kiers 3df9434a90 WIP: Message store
Signed-off-by: Jacob Kiers <jacob@jacobkiers.net>
2022-08-06 21:32:56 +02:00
6 changed files with 225 additions and 5 deletions

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
/target
/data
/output

159
Cargo.lock generated
View File

@ -48,6 +48,15 @@ version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
[[package]]
name = "bincode"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
dependencies = [
"serde",
]
[[package]]
name = "bitflags"
version = "1.3.2"
@ -75,6 +84,12 @@ version = "3.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3"
[[package]]
name = "byteorder"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
name = "cc"
version = "1.0.73"
@ -110,6 +125,39 @@ dependencies = [
"libc",
]
[[package]]
name = "crc32fast"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "045ebe27666471bb549370b4b0b3e51b07f56325befa4284db65fc89c02511b1"
dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils",
"memoffset",
"once_cell",
"scopeguard",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc"
dependencies = [
"cfg-if",
"once_cell",
]
[[package]]
name = "crypto-common"
version = "0.1.6"
@ -220,6 +268,25 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "fs2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "fxhash"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
dependencies = [
"byteorder",
]
[[package]]
name = "generic-array"
version = "0.14.6"
@ -260,6 +327,15 @@ dependencies = [
"nom",
]
[[package]]
name = "instant"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
dependencies = [
"cfg-if",
]
[[package]]
name = "js-sys"
version = "0.3.59"
@ -294,6 +370,16 @@ version = "0.2.127"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "505e71a4706fa491e9b1b55f51b95d4037d0821ee40131190475f692b35b009b"
[[package]]
name = "lock_api"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53"
dependencies = [
"autocfg",
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.17"
@ -319,6 +405,15 @@ version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
[[package]]
name = "memoffset"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
dependencies = [
"autocfg",
]
[[package]]
name = "never"
version = "0.1.0"
@ -331,11 +426,13 @@ version = "0.1.0"
dependencies = [
"atom_syndication",
"base16ct",
"bincode",
"chrono",
"imap",
"mail-parser",
"rustls-connector",
"sha2",
"sled",
]
[[package]]
@ -374,6 +471,31 @@ version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1"
[[package]]
name = "parking_lot"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
dependencies = [
"instant",
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216"
dependencies = [
"cfg-if",
"instant",
"libc",
"redox_syscall",
"smallvec",
"winapi",
]
[[package]]
name = "proc-macro2"
version = "1.0.43"
@ -402,6 +524,15 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "redox_syscall"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
"bitflags",
]
[[package]]
name = "regex"
version = "1.6.0"
@ -464,6 +595,12 @@ version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "sct"
version = "0.7.0"
@ -505,6 +642,28 @@ dependencies = [
"digest",
]
[[package]]
name = "sled"
version = "0.34.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"
dependencies = [
"crc32fast",
"crossbeam-epoch",
"crossbeam-utils",
"fs2",
"fxhash",
"libc",
"log",
"parking_lot",
]
[[package]]
name = "smallvec"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1"
[[package]]
name = "spin"
version = "0.5.2"

View File

@ -9,8 +9,10 @@ description = "Converts email newsletters to static HTML files"
[dependencies]
atom_syndication = "^0.11.0"
base16ct = { version = "^0.1.0", features = [ "alloc" ] }
bincode = "^1.3.3"
chrono = "^0.4"
imap = { version = "^2.4.1", default-features = false }
mail-parser = "^0.5.0"
rustls-connector = { version = "^0.16.1", default-features = false, features = [ "webpki-roots-certs", "quic" ] }
sha2 = "^0.10.2"
sled = "^0.34.7"

View File

@ -1,20 +1,23 @@
mod message_reader;
mod storage;
use std::{
fs::{File, OpenOptions},
io::Write,
path::{Path, PathBuf},
path::{Path, PathBuf}, error::Error,
};
use atom_syndication::{
ContentBuilder, Entry, EntryBuilder, Feed, FeedBuilder, Generator, LinkBuilder, Person,
};
use chrono::{DateTime, TimeZone, Utc};
use mail_parser::{HeaderValue, Message as MpMessage};
use sha2::{Digest, Sha256};
use message_reader::{EmailReader, TestMessagesReader};
use storage::Store;
pub struct Message {
uid: String,
@ -35,8 +38,8 @@ impl Message {
}
}
fn main() {
let dir = Path::new("data");
fn main() -> Result<(), Box<dyn Error>> {
let dir = Path::new("output");
if !dir.exists() {
std::fs::create_dir(&dir).expect("Could not create directory");
}
@ -44,9 +47,11 @@ fn main() {
let mut feed = build_atom_feed();
let mut reader = TestMessagesReader::new((&Path::new("tests/data")).to_path_buf());
let store = Store::load_database_for_mailbox("newsletters@kie.rs")?;
for msg in reader.read_rfc822_messages() {
println!("Processing message {}", msg.get_uid());
store.store_mail(&msg)?;
let parsed = msg.get_parsed().expect("A parsed messsage.");
@ -75,6 +80,8 @@ fn main() {
feed.set_updated(Utc::now());
let _ = feed.write_to(File::create(format!("{}/feed.atom", dir.display())).unwrap());
}
Ok(())
}
fn add_entry_to_feed(feed: &mut Feed, message: &Message, processed_html: &String) {
@ -165,7 +172,7 @@ fn build_atom_feed() -> Feed {
.build()
}
fn write_to_test_path(msg: &Message) {
fn _write_to_test_path(msg: &Message) {
let test_path: PathBuf = [
Path::new("tests/data"),
Path::new(&format!("{}.eml", &msg.get_uid())),

View File

@ -66,7 +66,7 @@ pub struct ImapReader {
}
impl ImapReader {
pub fn new(host: String, port: u16, username: String, password: String) -> Self {
pub fn _new(host: String, port: u16, username: String, password: String) -> Self {
ImapReader {
host,
port,

51
bin/src/storage.rs Normal file
View File

@ -0,0 +1,51 @@
use sled::{Db, Transactional};
use crate::Message;
use std::str::FromStr;
pub(crate) struct Store {
db: Db,
mailbox: String,
}
type ER = Result<(), sled::Error>;
type BR = Result<bool, sled::Error>;
impl Store {
pub fn load_database_for_mailbox<S: Into<String>>(mailbox: S) -> Result<Self, sled::Error> {
let db = sled::open("data/maildb")?;
Ok(Store {
db,
mailbox: mailbox.into(),
})
}
pub fn store_mail(&self, message: &Message) -> ER {
self.mb()?.insert(message.get_uid(), &*message.data)?;
Ok(())
}
pub fn has_mail<S: Into<String>>(&self, uid: S) -> BR {
self.mb()?.contains_key(uid.into())
}
pub fn mark_in_feed<S: Into<String>>(&self, uid: S) -> ER {
self.feed()?.insert(uid.into(), &[1])?;
Ok(())
}
pub fn is_in_feed<S: Into<String>>(&self, uid: S) -> BR {
match self.feed()?.get(uid.into())? {
Some(v) => Ok(bincode::deserialize(&v).expect("Cannot convert to bool")),
None => Ok(false),
}
}
fn mb(&self) -> Result<sled::Tree, sled::Error> {
self.db.open_tree(&self.mailbox)
}
fn feed(&self) -> Result<sled::Tree, sled::Error> {
self.db.open_tree("feed")
}
}