newsletter-to-web/bin/src/main.rs

// extern crate atom_syndication;
// extern crate imap;
// extern crate mail_parser;
// extern crate rustls_connector;
// extern crate sha2;

mod message_reader;

use std::{
    borrow::Cow,
    fs::{File, OpenOptions},
    io::Write,
    path::{Path, PathBuf},
};

use atom_syndication::{
    ContentBuilder, Entry, EntryBuilder, Feed, FeedBuilder, Generator, LinkBuilder, Person,
};
use chrono::{DateTime, FixedOffset, Offset, TimeZone, Utc};
use mail_parser::{HeaderName, HeaderValue, Message as MpMessage, RfcHeader};

use sha2::{Digest, Sha256};

use message_reader::{EmailReader, TestMessagesReader};

pub struct Message {
    uid: String,
    data: Vec<u8>,
}

impl Message {
    pub fn new(uid: String, data: Vec<u8>) -> Message {
        Message { uid, data }
    }

    pub(crate) fn get_parsed(&self) -> Option<MpMessage> {
        MpMessage::parse(&self.data)
    }

    pub fn get_uid(&self) -> &String {
        &self.uid
    }
}

fn main() {
    let dir = Path::new("data");
    if !dir.exists() {
        std::fs::create_dir(&dir).expect("Could not create directory");
    }

    let mut feed = build_atom_feed();

    let mut reader = TestMessagesReader::new((&Path::new("tests/data")).to_path_buf());

    for msg in reader.read_rfc822_messages() {
        println!("Processing message {}", msg.get_uid());

        let parsed = msg.get_parsed().expect("A parsed messsage.");

        let html_body = parsed.get_html_body(0).expect("Could not read html body");
        let processed_html = process_html(&html_body).expect("Could not process the HTML");
        let html_bytes = processed_html.as_bytes();

        let path = get_path(&parsed, &msg);
        let html_path: PathBuf = [dir, Path::new(&path)].iter().collect();
        println!("Storing to {}", &html_path.display());

        add_entry_to_feed(&mut feed, &msg, &processed_html);

        OpenOptions::new()
            .write(true)
            .create(true)
            .open(&html_path)
            .expect(format!("Could not open file '{}' for writing", &html_path.display()).as_str())
            .write_all(&html_bytes)
            .expect(format!("Could not write html to file '{}'.", &html_path.display()).as_str());

        println!();
    }

    if feed.entries.len() > 0 {
        feed.set_updated(Utc::now());
        let _ = feed.write_to(File::create(format!("{}/feed.atom", dir.display())).unwrap());
    }
}

fn add_entry_to_feed(feed: &mut Feed, message: &Message, processed_html: &String) {
    let parsed = message.get_parsed().unwrap();
    let date = parsed.get_date().expect("Could not extract date");
    let from = match parsed.get_from() {
        HeaderValue::Address(e) => e,
        _ => return,
    };

    let path = get_path(&parsed, message);

    let url = format!("https://newsletters.kiers.eu/{}", &path);

    let mut entry : Entry = Newsletter {
        author: Person {
            name: match &from.name {
                Some(n) => n.to_string(),
                _ => match &from.address {
                    Some(e) => e.to_string(),
                    _ => "".to_string(),
                },
            },
            email: match &from.address {
                Some(e) => Some(e.to_string()),
                _ => None,
            },
            uri: None,
        },
        title: parsed
            .get_subject()
            .expect("Expected a subject")
            .to_string(),
        content: Some(processed_html.clone()),
        id: url.clone(),
        published: Utc.timestamp(date.to_timestamp(), 0), //(format!("{}{}", &date.to_iso8601(), "+00:00").as_str()).`unwrap(),
        url: url,
    }
    .into();
    entry.set_updated(Utc.timestamp(date.to_timestamp(), 0));
    feed.entries.push(entry);
}

fn get_path(parsed: &MpMessage, msg: &Message) -> String {
    let date = parsed.get_date().expect("Could not extract date");
    let date_str = format!(
        "{:04}{:02}{:02}{:02}{:02}{:02}",
        &date.year, &date.month, &date.day, &date.hour, &date.minute, &date.second
    );

    let hash = base16ct::lower::encode_string(&Sha256::digest(
        &parsed.get_html_body(0).expect("Expected a body").as_bytes(),
    ));

    let uid: i32 = msg
        .get_uid()
        .parse()
        .expect("Could not convert message uid to an i32.");
    //format!("{}_{}_{}.html", &date_str, &file_name, msg.get_uid()).to_owned()
    format!("{:05}_{}_{}.html", uid, date_str, &hash).to_owned()
}

fn process_html(input: &str) -> Result<String, ()> {
    Ok(input.replace("src", "data-source"))
}

fn build_atom_feed() -> Feed {
    FeedBuilder::default()
        .title("JJKiers Newsletters")
        .id("https://newsletters.kiers.eu/feed.atom")
        .link(
            LinkBuilder::default()
                .href("https://newsletters.kiers.eu/")
                .rel("alternate")
                .build(),
        )
        .link(
            LinkBuilder::default()
                .href("https://newsletters.kiers.eu/feed.atom")
                .rel("self")
                .build(),
        )
        .generator(Generator {
            value: String::from("newsletter-to-web"),
            uri: None,
            version: Some(String::from("0.0.1")),
        })
        .build()
}

fn write_to_test_path(msg: &Message) {
    let test_path: PathBuf = [
        Path::new("tests/data"),
        Path::new(&format!("{}.eml", &msg.get_uid())),
    ]
    .iter()
    .collect();

    let _ = OpenOptions::new()
        .write(true)
        .create(true)
        .open(test_path)
        .expect("Could not open file fir writing")
        .write_all(&msg.data);
}

//#[derive(Serialize, Deserialize, Debug)]
struct Newsletter {
    id: String,
    url: String,
    title: String,
    content: Option<String>,
    author: Person,
    published: DateTime<Utc>,
}

impl From<Newsletter> for Entry {
    fn from(post: Newsletter) -> Self {
        let content = post.content.map(|v| {
            ContentBuilder::default()
                .value(v)
                .content_type(Some("html".to_string()))
                .build()
        });

        EntryBuilder::default()
            .title(post.title)
            .id(post.id)
            .published(Some(post.published.clone().into()))
            .author(post.author.into())
            .content(content)
            .link(
                LinkBuilder::default()
                    .href(post.url)
                    .rel("alternate")
                    .build(),
            )
            .build()
    }
}

// pub fn parse_datetime(s: &str) -> Option<DateTime<FixedOffset>> {
//     DateTime::<FixedOffset>::from(s)
//         .ok()
//         .map(|d| d.with_timezone(&Utc.fix()))
// }