Compare commits
1 Commits
master
...
message-st
Author | SHA1 | Date |
---|---|---|
Jacob Kiers | 3df9434a90 |
|
@ -1,2 +1,3 @@
|
||||||
/target
|
/target
|
||||||
/data
|
/data
|
||||||
|
/output
|
||||||
|
|
|
@ -48,6 +48,15 @@ version = "0.13.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
|
checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bincode"
|
||||||
|
version = "1.3.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
|
||||||
|
dependencies = [
|
||||||
|
"serde",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bitflags"
|
name = "bitflags"
|
||||||
version = "1.3.2"
|
version = "1.3.2"
|
||||||
|
@ -75,6 +84,12 @@ version = "3.10.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3"
|
checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "byteorder"
|
||||||
|
version = "1.4.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cc"
|
name = "cc"
|
||||||
version = "1.0.73"
|
version = "1.0.73"
|
||||||
|
@ -110,6 +125,39 @@ dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crc32fast"
|
||||||
|
version = "1.3.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crossbeam-epoch"
|
||||||
|
version = "0.9.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "045ebe27666471bb549370b4b0b3e51b07f56325befa4284db65fc89c02511b1"
|
||||||
|
dependencies = [
|
||||||
|
"autocfg",
|
||||||
|
"cfg-if",
|
||||||
|
"crossbeam-utils",
|
||||||
|
"memoffset",
|
||||||
|
"once_cell",
|
||||||
|
"scopeguard",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crossbeam-utils"
|
||||||
|
version = "0.8.11"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"once_cell",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "crypto-common"
|
name = "crypto-common"
|
||||||
version = "0.1.6"
|
version = "0.1.6"
|
||||||
|
@ -220,6 +268,25 @@ version = "1.0.7"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
|
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fs2"
|
||||||
|
version = "0.4.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"winapi",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fxhash"
|
||||||
|
version = "0.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
|
||||||
|
dependencies = [
|
||||||
|
"byteorder",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "generic-array"
|
name = "generic-array"
|
||||||
version = "0.14.6"
|
version = "0.14.6"
|
||||||
|
@ -260,6 +327,15 @@ dependencies = [
|
||||||
"nom",
|
"nom",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "instant"
|
||||||
|
version = "0.1.12"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "js-sys"
|
name = "js-sys"
|
||||||
version = "0.3.59"
|
version = "0.3.59"
|
||||||
|
@ -294,6 +370,16 @@ version = "0.2.127"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "505e71a4706fa491e9b1b55f51b95d4037d0821ee40131190475f692b35b009b"
|
checksum = "505e71a4706fa491e9b1b55f51b95d4037d0821ee40131190475f692b35b009b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "lock_api"
|
||||||
|
version = "0.4.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53"
|
||||||
|
dependencies = [
|
||||||
|
"autocfg",
|
||||||
|
"scopeguard",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "log"
|
name = "log"
|
||||||
version = "0.4.17"
|
version = "0.4.17"
|
||||||
|
@ -319,6 +405,15 @@ version = "2.5.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
|
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "memoffset"
|
||||||
|
version = "0.6.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
|
||||||
|
dependencies = [
|
||||||
|
"autocfg",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "never"
|
name = "never"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
|
@ -331,11 +426,13 @@ version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"atom_syndication",
|
"atom_syndication",
|
||||||
"base16ct",
|
"base16ct",
|
||||||
|
"bincode",
|
||||||
"chrono",
|
"chrono",
|
||||||
"imap",
|
"imap",
|
||||||
"mail-parser",
|
"mail-parser",
|
||||||
"rustls-connector",
|
"rustls-connector",
|
||||||
"sha2",
|
"sha2",
|
||||||
|
"sled",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
@ -374,6 +471,31 @@ version = "1.13.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1"
|
checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "parking_lot"
|
||||||
|
version = "0.11.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
|
||||||
|
dependencies = [
|
||||||
|
"instant",
|
||||||
|
"lock_api",
|
||||||
|
"parking_lot_core",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "parking_lot_core"
|
||||||
|
version = "0.8.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"instant",
|
||||||
|
"libc",
|
||||||
|
"redox_syscall",
|
||||||
|
"smallvec",
|
||||||
|
"winapi",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "proc-macro2"
|
name = "proc-macro2"
|
||||||
version = "1.0.43"
|
version = "1.0.43"
|
||||||
|
@ -402,6 +524,15 @@ dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "redox_syscall"
|
||||||
|
version = "0.2.16"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "regex"
|
name = "regex"
|
||||||
version = "1.6.0"
|
version = "1.6.0"
|
||||||
|
@ -464,6 +595,12 @@ version = "1.0.11"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
|
checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "scopeguard"
|
||||||
|
version = "1.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "sct"
|
name = "sct"
|
||||||
version = "0.7.0"
|
version = "0.7.0"
|
||||||
|
@ -505,6 +642,28 @@ dependencies = [
|
||||||
"digest",
|
"digest",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "sled"
|
||||||
|
version = "0.34.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"
|
||||||
|
dependencies = [
|
||||||
|
"crc32fast",
|
||||||
|
"crossbeam-epoch",
|
||||||
|
"crossbeam-utils",
|
||||||
|
"fs2",
|
||||||
|
"fxhash",
|
||||||
|
"libc",
|
||||||
|
"log",
|
||||||
|
"parking_lot",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "smallvec"
|
||||||
|
version = "1.9.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "spin"
|
name = "spin"
|
||||||
version = "0.5.2"
|
version = "0.5.2"
|
||||||
|
|
|
@ -9,8 +9,10 @@ description = "Converts email newsletters to static HTML files"
|
||||||
[dependencies]
|
[dependencies]
|
||||||
atom_syndication = "^0.11.0"
|
atom_syndication = "^0.11.0"
|
||||||
base16ct = { version = "^0.1.0", features = [ "alloc" ] }
|
base16ct = { version = "^0.1.0", features = [ "alloc" ] }
|
||||||
|
bincode = "^1.3.3"
|
||||||
chrono = "^0.4"
|
chrono = "^0.4"
|
||||||
imap = { version = "^2.4.1", default-features = false }
|
imap = { version = "^2.4.1", default-features = false }
|
||||||
mail-parser = "^0.5.0"
|
mail-parser = "^0.5.0"
|
||||||
rustls-connector = { version = "^0.16.1", default-features = false, features = [ "webpki-roots-certs", "quic" ] }
|
rustls-connector = { version = "^0.16.1", default-features = false, features = [ "webpki-roots-certs", "quic" ] }
|
||||||
sha2 = "^0.10.2"
|
sha2 = "^0.10.2"
|
||||||
|
sled = "^0.34.7"
|
|
@ -1,20 +1,23 @@
|
||||||
mod message_reader;
|
mod message_reader;
|
||||||
|
mod storage;
|
||||||
|
|
||||||
use std::{
|
use std::{
|
||||||
fs::{File, OpenOptions},
|
fs::{File, OpenOptions},
|
||||||
io::Write,
|
io::Write,
|
||||||
path::{Path, PathBuf},
|
path::{Path, PathBuf}, error::Error,
|
||||||
};
|
};
|
||||||
|
|
||||||
use atom_syndication::{
|
use atom_syndication::{
|
||||||
ContentBuilder, Entry, EntryBuilder, Feed, FeedBuilder, Generator, LinkBuilder, Person,
|
ContentBuilder, Entry, EntryBuilder, Feed, FeedBuilder, Generator, LinkBuilder, Person,
|
||||||
};
|
};
|
||||||
|
|
||||||
use chrono::{DateTime, TimeZone, Utc};
|
use chrono::{DateTime, TimeZone, Utc};
|
||||||
use mail_parser::{HeaderValue, Message as MpMessage};
|
use mail_parser::{HeaderValue, Message as MpMessage};
|
||||||
|
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
|
|
||||||
use message_reader::{EmailReader, TestMessagesReader};
|
use message_reader::{EmailReader, TestMessagesReader};
|
||||||
|
use storage::Store;
|
||||||
|
|
||||||
pub struct Message {
|
pub struct Message {
|
||||||
uid: String,
|
uid: String,
|
||||||
|
@ -35,8 +38,8 @@ impl Message {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn main() {
|
fn main() -> Result<(), Box<dyn Error>> {
|
||||||
let dir = Path::new("data");
|
let dir = Path::new("output");
|
||||||
if !dir.exists() {
|
if !dir.exists() {
|
||||||
std::fs::create_dir(&dir).expect("Could not create directory");
|
std::fs::create_dir(&dir).expect("Could not create directory");
|
||||||
}
|
}
|
||||||
|
@ -44,9 +47,11 @@ fn main() {
|
||||||
let mut feed = build_atom_feed();
|
let mut feed = build_atom_feed();
|
||||||
|
|
||||||
let mut reader = TestMessagesReader::new((&Path::new("tests/data")).to_path_buf());
|
let mut reader = TestMessagesReader::new((&Path::new("tests/data")).to_path_buf());
|
||||||
|
let store = Store::load_database_for_mailbox("newsletters@kie.rs")?;
|
||||||
|
|
||||||
for msg in reader.read_rfc822_messages() {
|
for msg in reader.read_rfc822_messages() {
|
||||||
println!("Processing message {}", msg.get_uid());
|
println!("Processing message {}", msg.get_uid());
|
||||||
|
store.store_mail(&msg)?;
|
||||||
|
|
||||||
let parsed = msg.get_parsed().expect("A parsed messsage.");
|
let parsed = msg.get_parsed().expect("A parsed messsage.");
|
||||||
|
|
||||||
|
@ -75,6 +80,8 @@ fn main() {
|
||||||
feed.set_updated(Utc::now());
|
feed.set_updated(Utc::now());
|
||||||
let _ = feed.write_to(File::create(format!("{}/feed.atom", dir.display())).unwrap());
|
let _ = feed.write_to(File::create(format!("{}/feed.atom", dir.display())).unwrap());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn add_entry_to_feed(feed: &mut Feed, message: &Message, processed_html: &String) {
|
fn add_entry_to_feed(feed: &mut Feed, message: &Message, processed_html: &String) {
|
||||||
|
@ -165,7 +172,7 @@ fn build_atom_feed() -> Feed {
|
||||||
.build()
|
.build()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn write_to_test_path(msg: &Message) {
|
fn _write_to_test_path(msg: &Message) {
|
||||||
let test_path: PathBuf = [
|
let test_path: PathBuf = [
|
||||||
Path::new("tests/data"),
|
Path::new("tests/data"),
|
||||||
Path::new(&format!("{}.eml", &msg.get_uid())),
|
Path::new(&format!("{}.eml", &msg.get_uid())),
|
||||||
|
|
|
@ -66,7 +66,7 @@ pub struct ImapReader {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ImapReader {
|
impl ImapReader {
|
||||||
pub fn new(host: String, port: u16, username: String, password: String) -> Self {
|
pub fn _new(host: String, port: u16, username: String, password: String) -> Self {
|
||||||
ImapReader {
|
ImapReader {
|
||||||
host,
|
host,
|
||||||
port,
|
port,
|
||||||
|
|
|
@ -0,0 +1,51 @@
|
||||||
|
use sled::{Db, Transactional};
|
||||||
|
|
||||||
|
use crate::Message;
|
||||||
|
use std::str::FromStr;
|
||||||
|
|
||||||
|
pub(crate) struct Store {
|
||||||
|
db: Db,
|
||||||
|
mailbox: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
type ER = Result<(), sled::Error>;
|
||||||
|
type BR = Result<bool, sled::Error>;
|
||||||
|
|
||||||
|
impl Store {
|
||||||
|
pub fn load_database_for_mailbox<S: Into<String>>(mailbox: S) -> Result<Self, sled::Error> {
|
||||||
|
let db = sled::open("data/maildb")?;
|
||||||
|
Ok(Store {
|
||||||
|
db,
|
||||||
|
mailbox: mailbox.into(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn store_mail(&self, message: &Message) -> ER {
|
||||||
|
self.mb()?.insert(message.get_uid(), &*message.data)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn has_mail<S: Into<String>>(&self, uid: S) -> BR {
|
||||||
|
self.mb()?.contains_key(uid.into())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn mark_in_feed<S: Into<String>>(&self, uid: S) -> ER {
|
||||||
|
self.feed()?.insert(uid.into(), &[1])?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_in_feed<S: Into<String>>(&self, uid: S) -> BR {
|
||||||
|
match self.feed()?.get(uid.into())? {
|
||||||
|
Some(v) => Ok(bincode::deserialize(&v).expect("Cannot convert to bool")),
|
||||||
|
None => Ok(false),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn mb(&self) -> Result<sled::Tree, sled::Error> {
|
||||||
|
self.db.open_tree(&self.mailbox)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn feed(&self) -> Result<sled::Tree, sled::Error> {
|
||||||
|
self.db.open_tree("feed")
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue