WIP: Message store
Signed-off-by: Jacob Kiers <jacob@jacobkiers.net>
This commit is contained in:
parent
f552e86945
commit
3df9434a90
|
@ -1,2 +1,3 @@
|
|||
/target
|
||||
/data
|
||||
/output
|
||||
|
|
|
@ -48,6 +48,15 @@ version = "0.13.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
|
||||
|
||||
[[package]]
|
||||
name = "bincode"
|
||||
version = "1.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.3.2"
|
||||
|
@ -75,6 +84,12 @@ version = "3.10.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.73"
|
||||
|
@ -110,6 +125,39 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "045ebe27666471bb549370b4b0b3e51b07f56325befa4284db65fc89c02511b1"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
"memoffset",
|
||||
"once_cell",
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.6"
|
||||
|
@ -220,6 +268,25 @@ version = "1.0.7"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
|
||||
|
||||
[[package]]
|
||||
name = "fs2"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fxhash"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.6"
|
||||
|
@ -260,6 +327,15 @@ dependencies = [
|
|||
"nom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "instant"
|
||||
version = "0.1.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.59"
|
||||
|
@ -294,6 +370,16 @@ version = "0.2.127"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "505e71a4706fa491e9b1b55f51b95d4037d0821ee40131190475f692b35b009b"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.17"
|
||||
|
@ -319,6 +405,15 @@ version = "2.5.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.6.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "never"
|
||||
version = "0.1.0"
|
||||
|
@ -331,11 +426,13 @@ version = "0.1.0"
|
|||
dependencies = [
|
||||
"atom_syndication",
|
||||
"base16ct",
|
||||
"bincode",
|
||||
"chrono",
|
||||
"imap",
|
||||
"mail-parser",
|
||||
"rustls-connector",
|
||||
"sha2",
|
||||
"sled",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
@ -374,6 +471,31 @@ version = "1.13.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.11.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
|
||||
dependencies = [
|
||||
"instant",
|
||||
"lock_api",
|
||||
"parking_lot_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.8.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"instant",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"smallvec",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.43"
|
||||
|
@ -402,6 +524,15 @@ dependencies = [
|
|||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.6.0"
|
||||
|
@ -464,6 +595,12 @@ version = "1.0.11"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
||||
|
||||
[[package]]
|
||||
name = "sct"
|
||||
version = "0.7.0"
|
||||
|
@ -505,6 +642,28 @@ dependencies = [
|
|||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sled"
|
||||
version = "0.34.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
"fs2",
|
||||
"fxhash",
|
||||
"libc",
|
||||
"log",
|
||||
"parking_lot",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1"
|
||||
|
||||
[[package]]
|
||||
name = "spin"
|
||||
version = "0.5.2"
|
||||
|
|
|
@ -9,8 +9,10 @@ description = "Converts email newsletters to static HTML files"
|
|||
[dependencies]
|
||||
atom_syndication = "^0.11.0"
|
||||
base16ct = { version = "^0.1.0", features = [ "alloc" ] }
|
||||
bincode = "^1.3.3"
|
||||
chrono = "^0.4"
|
||||
imap = { version = "^2.4.1", default-features = false }
|
||||
mail-parser = "^0.5.0"
|
||||
rustls-connector = { version = "^0.16.1", default-features = false, features = [ "webpki-roots-certs", "quic" ] }
|
||||
sha2 = "^0.10.2"
|
||||
sled = "^0.34.7"
|
|
@ -1,20 +1,23 @@
|
|||
mod message_reader;
|
||||
mod storage;
|
||||
|
||||
use std::{
|
||||
fs::{File, OpenOptions},
|
||||
io::Write,
|
||||
path::{Path, PathBuf},
|
||||
path::{Path, PathBuf}, error::Error,
|
||||
};
|
||||
|
||||
use atom_syndication::{
|
||||
ContentBuilder, Entry, EntryBuilder, Feed, FeedBuilder, Generator, LinkBuilder, Person,
|
||||
};
|
||||
|
||||
use chrono::{DateTime, TimeZone, Utc};
|
||||
use mail_parser::{HeaderValue, Message as MpMessage};
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use message_reader::{EmailReader, TestMessagesReader};
|
||||
use storage::Store;
|
||||
|
||||
pub struct Message {
|
||||
uid: String,
|
||||
|
@ -35,8 +38,8 @@ impl Message {
|
|||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let dir = Path::new("data");
|
||||
fn main() -> Result<(), Box<dyn Error>> {
|
||||
let dir = Path::new("output");
|
||||
if !dir.exists() {
|
||||
std::fs::create_dir(&dir).expect("Could not create directory");
|
||||
}
|
||||
|
@ -44,9 +47,11 @@ fn main() {
|
|||
let mut feed = build_atom_feed();
|
||||
|
||||
let mut reader = TestMessagesReader::new((&Path::new("tests/data")).to_path_buf());
|
||||
let store = Store::load_database_for_mailbox("newsletters@kie.rs")?;
|
||||
|
||||
for msg in reader.read_rfc822_messages() {
|
||||
println!("Processing message {}", msg.get_uid());
|
||||
store.store_mail(&msg)?;
|
||||
|
||||
let parsed = msg.get_parsed().expect("A parsed messsage.");
|
||||
|
||||
|
@ -75,6 +80,8 @@ fn main() {
|
|||
feed.set_updated(Utc::now());
|
||||
let _ = feed.write_to(File::create(format!("{}/feed.atom", dir.display())).unwrap());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn add_entry_to_feed(feed: &mut Feed, message: &Message, processed_html: &String) {
|
||||
|
@ -165,7 +172,7 @@ fn build_atom_feed() -> Feed {
|
|||
.build()
|
||||
}
|
||||
|
||||
fn write_to_test_path(msg: &Message) {
|
||||
fn _write_to_test_path(msg: &Message) {
|
||||
let test_path: PathBuf = [
|
||||
Path::new("tests/data"),
|
||||
Path::new(&format!("{}.eml", &msg.get_uid())),
|
||||
|
|
|
@ -66,7 +66,7 @@ pub struct ImapReader {
|
|||
}
|
||||
|
||||
impl ImapReader {
|
||||
pub fn new(host: String, port: u16, username: String, password: String) -> Self {
|
||||
pub fn _new(host: String, port: u16, username: String, password: String) -> Self {
|
||||
ImapReader {
|
||||
host,
|
||||
port,
|
||||
|
|
|
@ -0,0 +1,51 @@
|
|||
use sled::{Db, Transactional};
|
||||
|
||||
use crate::Message;
|
||||
use std::str::FromStr;
|
||||
|
||||
pub(crate) struct Store {
|
||||
db: Db,
|
||||
mailbox: String,
|
||||
}
|
||||
|
||||
type ER = Result<(), sled::Error>;
|
||||
type BR = Result<bool, sled::Error>;
|
||||
|
||||
impl Store {
|
||||
pub fn load_database_for_mailbox<S: Into<String>>(mailbox: S) -> Result<Self, sled::Error> {
|
||||
let db = sled::open("data/maildb")?;
|
||||
Ok(Store {
|
||||
db,
|
||||
mailbox: mailbox.into(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn store_mail(&self, message: &Message) -> ER {
|
||||
self.mb()?.insert(message.get_uid(), &*message.data)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn has_mail<S: Into<String>>(&self, uid: S) -> BR {
|
||||
self.mb()?.contains_key(uid.into())
|
||||
}
|
||||
|
||||
pub fn mark_in_feed<S: Into<String>>(&self, uid: S) -> ER {
|
||||
self.feed()?.insert(uid.into(), &[1])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn is_in_feed<S: Into<String>>(&self, uid: S) -> BR {
|
||||
match self.feed()?.get(uid.into())? {
|
||||
Some(v) => Ok(bincode::deserialize(&v).expect("Cannot convert to bool")),
|
||||
None => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
fn mb(&self) -> Result<sled::Tree, sled::Error> {
|
||||
self.db.open_tree(&self.mailbox)
|
||||
}
|
||||
|
||||
fn feed(&self) -> Result<sled::Tree, sled::Error> {
|
||||
self.db.open_tree("feed")
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue