Add changelog and relase v0.2.4

Signed-off-by: Jacob Kiers <code@kiers.eu>

.cargo/config.toml

@@ -0,0 +1,3 @@
+[profile.release]
+lto = "thin"
+strip = true

.drone.jsonnet

@@ -1,42 +1,41 @@
 local executableName = 'newsletter-to-web';
+local build_image = 'img.kie.rs/jjkiers/rust-crossbuild:rust1.79.0-zig0.11.0-zig';
 
 local archs = [
-  //  { target: 'aarch64-unknown-linux-gnu', short: 'arm64-gnu' },
   { target: 'aarch64-unknown-linux-musl', short: 'arm64-musl' },
   { target: 'x86_64-pc-windows-gnu', short: 'windows' },
-  //  { target: 'x86_64-unknown-linux-gnu', short: 'amd64-gnu' },
   { target: 'x86_64-unknown-linux-musl', short: 'amd64-musl' },
 ];
 
 local getStepName(arch) = 'Build for ' + arch.short;
 
-local builtExecutableName(arch) = executableName + if std.startsWith(arch.short, 'windows') then '.exe' else '';
-local targetExecutableName(arch) = executableName + if std.startsWith(arch.short, 'windows') then '.exe' else '-' + arch.short;
+local builtExecutableName(arch) = executableName + if std.length(std.findSubstr(arch.short, 'windows')) > 0 then '.exe' else '';
+local targetExecutableName(arch) = executableName + '-' + arch.target + if std.length(std.findSubstr(arch.short, 'windows')) > 0 then '.exe' else '';
+
+local getVolumeName(arch) = 'target-' + arch.target;
+local getLocalVolumes(arch) = [
+  {
+    name: getVolumeName(arch),
+    temp: {},
+  }
+  for arch in archs
+];
 
 local add_build_steps() = [
   {
     name: getStepName(arch),
-    image: 'img.kie.rs/jjkiers/rust-dind-cross:1.62-slim',
-    volumes: [
-      {
-        name: 'dockersock',
-        path: '/var/run',
-      },
-      {
-        name: 'rustup',
-        path: '/usr/local/rustup',
-      },
-    ],
+    image: build_image,
     commands: [
       'echo Hello World from Jsonnet on ' + arch.target + '!',
-      'cross build --release --target ' + arch.target,
+      'cargo zigbuild --release --target ' + arch.target,
       'cp target/' + arch.target + '/release/' + builtExecutableName(arch) + ' artifacts/' + targetExecutableName(arch),
       'rm -rf target/' + arch.target + '/release/*',
     ],
-    environment: {
-      CROSS_REMOTE: true,
-    },
-    depends_on: ['Wait for Docker'],
+    depends_on: ['Prepare'],
+    volumes: [{
+      name: getVolumeName(arch),
+      path: '/drone/src/target',
+    }],
   }
   for arch in archs
 ];
@@ -50,24 +49,20 @@ local add_build_steps() = [
   },
   steps:
     [{
-      name: 'Wait for Docker',
-      image: 'img.kie.rs/jjkiers/rust-dind-cross:1.65-slim',
+      name: 'Prepare',
+      image: build_image,
       commands: [
-        'while ! docker image ls; do sleep 1; done',
-        'docker info',
-        'docker pull hello-world:latest',
         'mkdir artifacts',
+        'echo Using image: ' + build_image,
+        'cargo --version',
+        'rustc --version',
       ],
-      volumes: [{
-        name: 'dockersock',
-        path: '/var/run',
-      }],
     }] +
     add_build_steps() +
     [
       {
         name: 'Show built artifacts',
-        image: 'img.kie.rs/jjkiers/rust-dind-cross:1.62-slim',
+        image: build_image,
         commands: [
           'ls -lah artifacts',
         ],
@@ -91,40 +86,7 @@ local add_build_steps() = [
       },
     ],
 
-  services: [{
-    name: 'docker',
-    image: 'docker:dind',
-    privileged: true,
-    volumes: [
-      {
-        name: 'dockersock',
-        path: '/var/run',
-      },
-      {
-        name: 'docker-storage',
-        path: '/var/lib/docker',
-      },
-    ],
-  }],
-
-  volumes: [
-    {
-      name: 'dockersock',
-      temp: {},
-    },
-    {
-      name: 'docker-storage',
-      host: {
-        path: '/srv/drone/docker-dind-rust',
-      },
-    },
-    {
-      name: 'rustup',
-      host: {
-        path: '/srv/drone/rustup',
-      },
-    },
-  ],
+  volumes: getLocalVolumes(archs),
 
   image_pull_secrets: ['docker_private_repo'],
 }

CHANGELOG.md

@@ -5,6 +5,38 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Changed
+
+## [0.2.4] - 2024-06-27
+
+### Added
+
+* Added `--max-items/-n` parameter to `build-feed` subcommand to limit number of entries in the feed.
+
+### Changed
+
+* Added protection features to example systemd service file
+* Refactored the code a bit
+
+## [0.2.3] - 2022-12-29
+
+### Added
+
+* Added `update` subcommand to update to the latest version.
+
+### Fixed
+
+* Truncate feed file before writing, to prevent corruption from leftover data.
+* Ensure the feed file name is part of the self URL. This was still hardcoded to `feed.atom`.
+
+## [0.2.2] - 2022-12-16
+
+### Changed
+
+* Updated build pipeline to generate much smaller binaries
+
 ## [0.2.1] - 2022-12-13
 
 ### Changed
@@ -22,4 +54,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
-* By default the name of the feed is now feed.xml instead of feed.atom.
+* By default, the name of the feed is now feed.xml instead of feed.atom.

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "newsletter-to-web"
-version = "0.2.0"
+version = "0.2.4"
 edition = "2021"
 description = "Converts email newsletters to static HTML files"
 homepage = "https://code.kiers.eu/newsletter-to-web/newsletter-to-web"
@@ -11,11 +11,13 @@ authors = [
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-atom_syndication = "^0.11.0"
-base16ct = { version = "^0.1.0", features = [ "alloc" ] }
-chrono = "^0.4"
-clap = { version = "^4.0.22", features = [ "derive" ] }
-imap = { version = "^2.4.1", default-features = false }
-mail-parser = "^0.8.0"
-rustls-connector = { version = "^0.16.1", default-features = false, features = [ "webpki-roots-certs", "quic" ] }
-sha2 = "^0.10.2"
+atom_syndication = "0.12.0"
+base16ct = { version = "0.2.0", features = [ "alloc" ] }
+chrono = "0.4.23"
+clap = { version = "4.2.7", features = [ "derive" ] }
+imap = { version = "3.0.0-alpha.10", default-features = false }
+mail-parser = "0.8.2"
+rustls-connector = { version = "0.17.0", default-features = false, features = [ "webpki-roots-certs" ] }
+self_update = { version = "0.36.0", default-features = false, features = ["rustls"] }
+sha2 = "0.10.6"
+reqwest = { version = "0.11.14", default-features = false, features = ["rustls", "blocking"] }

README.md

@@ -1,6 +1,41 @@
-# Newsletter 2 Web
+# Newsletter to Web
 
-Converts a newsletter to static HTML files.
+Converts a newsletter to and Atom feed and static HTML files.
+
+## Usage
+
+Get the latest release [from the releases page](https://code.kiers.eu/newsletter-to-web/newsletter-to-web/releases/latest).
+
+### Getting help
+
+For help, use 
+
+* `newsletter-to-web help`
+* `newsletter-to-web help <subcommand>`.
+
+### Basic usage
+
+First, download all messages from the IMAP mail server 
+and store them in the `data/` directory:
+
+```sh
+newsletter-to-web fetch-from-imap -s <imap.example.com> -u <email@example.com> -p <password>
+```
+
+Then, convert them to an Atom feed, using
+`newsletters.example.com` as the base domain:
+
+```sh
+newsletter-to-web --include-html build-feed newsletters.example.org
+```
+
+This will put the output in the `output/` directory. The Atom
+feed will be in `output/feed.xml`, together with a very simple
+`index.html` file pointing to the feed. It will also add an HTML
+file for every email with the HTML content.
+
+The feed will already contain the full HTML, so it can easily be
+read from a feed reader.
 
 ## Features
 

resources/feed.xsl

@@ -26,6 +26,7 @@
         </section>
         <section>
           <h2>Recent Items</h2>
+          <p>Last updated on <xsl:apply-templates select="atom:feed/atom:updated" /></p>
           <xsl:apply-templates select="atom:feed/atom:entry" />
         </section>
       </body>

src/cli.rs

@@ -1,6 +1,6 @@
 use std::path::PathBuf;
 
-use clap::{Parser, Subcommand};
+use clap::{Args, Parser, Subcommand};
 
 #[derive(Parser)]
 #[clap(author, version, about, long_about = None)]
@@ -9,35 +9,44 @@ pub(crate) struct Cli {
     pub command: Command,
 }
 
+#[derive(Args)]
+pub(crate) struct ImapSettings {
+    #[clap(short, long, value_parser)]
+    pub server: String,
+    #[clap(long, value_parser, default_value_t = 993)]
+    pub port: u16,
+    #[clap(short, long, value_parser)]
+    pub username: String,
+    #[clap(short, long, value_parser)]
+    pub password: String,
+}
+
+#[derive(Args)]
+pub(crate) struct FeedBuilderSettings {
+    /// Host name hosting the feed
+    pub hostname: String,
+    /// Feed file
+    #[clap(value_parser, default_value = "output/feed.xml")]
+    pub filename: PathBuf,
+    /// Create an HTML file for each message
+    #[clap(short, long, value_parser, default_value_t = false)]
+    pub include_html: bool,
+    /// Maximum number of items in the feed (default: unlimited)
+    #[clap(short = 'n', long, value_parser)]
+    pub max_items: Option<usize>,
+}
+
 #[derive(Subcommand)]
 pub(crate) enum Command {
     /// Fetch emails from an IMAP server
-    FetchFromImap {
-        #[clap(short, long, value_parser)]
-        server: String,
-        #[clap(long, value_parser, default_value_t = 993)]
-        port: u16,
-        #[clap(short, long, value_parser)]
-        username: String,
-        #[clap(short, long, value_parser)]
-        password: String,
-    },
+    FetchFromImap(ImapSettings),
     /// Fetch an email from a .eml file
     FetchFromFile {
         #[clap(value_parser)]
         filename: PathBuf,
     },
     /// Build an ATOM feed containing the full contents of the email
-    BuildFeed {
-        /// Host name hosting the feed
-        hostname: String,
-        /// Feed file
-        #[clap(value_parser, default_value = "output/feed.xml")]
-        filename: PathBuf,
-        /// Create an HTML file for each message
-        #[clap(short, long, value_parser, default_value_t = false)]
-        include_html: bool,
-    },
+    BuildFeed(FeedBuilderSettings),
     /// Exports the emails as HTML files
     ExportHtml {
         /// The directory in which the emails will be stored
@@ -50,4 +59,6 @@ pub(crate) enum Command {
         #[clap(value_parser, default_value = "output/")]
         directory: PathBuf,
     },
+    /// Update to the latest version
+    Update,
 }

src/command.rs

@@ -0,0 +1 @@
+pub(crate) mod update;

src/command/update.rs

@@ -0,0 +1,17 @@
+use self_update::cargo_crate_version;
+use std::error::Error;
+
+pub(crate) fn self_update() -> Result<(), Box<dyn Error>> {
+    let backend = self_update::backends::gitea::Update::configure()
+        .with_host("https://code.kiers.eu")
+        .repo_owner("newsletter-to-web")
+        .repo_name("newsletter-to-web")
+        .bin_name("newsletter-to-web")
+        .show_download_progress(true)
+        .current_version(cargo_crate_version!())
+        .build()?;
+
+    let status = backend.update()?;
+    println!("Update status: `{}`!", status.version());
+    Ok(())
+}

src/feed.rs

@@ -2,6 +2,7 @@ use crate::Message;
 
 use atom_syndication::{
     ContentBuilder, Entry, EntryBuilder, Feed, FeedBuilder, Generator, LinkBuilder, Person,
+    WriteConfig,
 };
 use chrono::{DateTime, TimeZone, Utc};
 
@@ -10,8 +11,8 @@ use mail_parser::HeaderValue;
 pub(crate) fn add_entry_to_feed(
     feed: &mut Feed,
     message: &Message,
-    processed_html: &String,
-    hostname: &String,
+    processed_html: &str,
+    hostname: &str,
     include_html: bool,
 ) {
     let parsed = message.get_parsed().unwrap();
@@ -34,19 +35,13 @@ pub(crate) fn add_entry_to_feed(
                     _ => "".to_string(),
                 },
             },
-            email: match &from.address {
-                Some(e) => Some(e.to_string()),
-                _ => None,
-            },
+            email: from.address.as_ref().map(|e| e.to_string()),
             uri: None,
         },
-        title: parsed
-            .subject()
-            .expect("Expected a subject")
-            .to_string(),
-        content: Some(processed_html.clone()),
+        title: parsed.subject().expect("Expected a subject").to_string(),
+        content: Some(processed_html.to_owned()),
         id: url.clone(),
-        published: Utc.timestamp_opt(date.to_timestamp(), 0).unwrap(), //(format!("{}{}", &date.to_iso8601(), "+00:00").as_str()).`unwrap(),
+        published: Utc.timestamp_opt(date.to_timestamp(), 0).unwrap(),
         url: match include_html {
             true => url,
             false => "".to_string(),
@@ -57,19 +52,20 @@ pub(crate) fn add_entry_to_feed(
     feed.entries.push(entry);
 }
 
-pub(crate) fn build_atom_feed(hostname: &String) -> Feed {
+pub(crate) fn build_atom_feed(hostname: &String, feed_file: &str) -> Feed {
+    let feed_url = format!("https://{hostname}/{feed_file}");
     FeedBuilder::default()
         .title("JJKiers Newsletters")
-        .id(format!("https://{}/feed.atom", hostname))
+        .id(&feed_url)
         .link(
             LinkBuilder::default()
-                .href(format!("https://{}/", hostname))
+                .href(format!("https://{hostname}/"))
                 .rel("alternate".to_string())
                 .build(),
         )
         .link(
             LinkBuilder::default()
-                .href(format!("https://{}/feed.atom", hostname))
+                .href(&feed_url)
                 .rel("self".to_string())
                 .build(),
         )
@@ -81,6 +77,19 @@ pub(crate) fn build_atom_feed(hostname: &String) -> Feed {
         .build()
 }
 
+pub(crate) fn write_feed<W: std::io::Write>(
+    feed: Feed,
+    mut out: W,
+) -> Result<W, atom_syndication::Error> {
+    let _ = writeln!(out, r#"<?xml version="1.0"?>"#);
+    let _ = writeln!(out, r#"<?xml-stylesheet href="feed.xsl" type="text/xsl"?>"#);
+    let config = WriteConfig {
+        write_document_declaration: false,
+        ..Default::default()
+    };
+    feed.write_with_config(out, config)
+}
+
 //#[derive(Serialize, Deserialize, Debug)]
 pub(crate) struct Newsletter {
     id: String,
@@ -104,8 +113,8 @@ impl From<Newsletter> for Entry {
 
         eb.title(post.title)
             .id(post.id)
-            .published(Some(post.published.clone().into()))
-            .author(post.author.into())
+            .published(Some(post.published.into()))
+            .author(post.author)
             .content(content);
 
         if post.url.len() > 1 {

src/main.rs

@@ -1,7 +1,7 @@
-#[warn(missing_docs)]
+#[deny(missing_docs)]
 #[doc = include_str!("../README.md")]
-
 mod cli;
+mod command;
 mod feed;
 mod message;
 mod message_reader;
@@ -20,32 +20,34 @@ use std::{
 
 pub(crate) use message::Message;
 
-const INDEX_HTML: & 'static str = include_str!("../resources/index.html");
-const FEED_STYLESHEET: & 'static str = include_str!("../resources/feed.xsl");
+const INDEX_HTML: &str = include_str!("../resources/index.html");
+const FEED_STYLESHEET: &str = include_str!("../resources/feed.xsl");
 
 fn main() -> Result<(), Box<dyn Error>> {
     let cli = cli::Cli::parse();
 
     let data_directory = "data";
 
-    let result = match &cli.command {
-        cli::Command::FetchFromImap {
-            server,
-            port,
-            username,
-            password,
-        } => fetch_from_imap(
+    match &cli.command {
+        cli::Command::FetchFromImap(imap_settings) => fetch_from_imap(
             data_directory,
-            server.to_owned(),
-            *port,
-            username.to_owned(),
-            password.to_owned(),
+            imap_settings.server.to_owned(),
+            imap_settings.port,
+            imap_settings.username.to_owned(),
+            imap_settings.password.to_owned(),
         ),
-        cli::Command::BuildFeed { filename , hostname, include_html } => build_feed(&filename, hostname.to_owned(), *include_html),
-        _ => unimplemented!("This method is not yet implemented."),
-    };
 
-    result
+        cli::Command::BuildFeed(settings) => build_feed(
+            &settings.filename,
+            &settings.hostname,
+            settings.include_html,
+            settings.max_items
+        ),
+
+        cli::Command::Update => command::update::self_update(),
+
+        _ => unimplemented!("This method is not yet implemented."),
+    }
 }
 
 fn create_directory<P: AsRef<Path>>(dir: P) -> Result<(), std::io::Error> {
@@ -56,7 +58,12 @@ fn create_directory<P: AsRef<Path>>(dir: P) -> Result<(), std::io::Error> {
     Ok(())
 }
 
-fn build_feed(filename: &PathBuf, hostname: String, include_html: bool) -> Result<(), Box<dyn Error>> {
+fn build_feed(
+    filename: &PathBuf,
+    hostname: &String,
+    include_html: bool,
+    max_items_in_feed: Option<usize>
+) -> Result<(), Box<dyn Error>> {
     let dir = filename.parent().ok_or(format!(
         "Could not get parent directory of {}",
         filename.display()
@@ -70,11 +77,19 @@ fn build_feed(filename: &PathBuf, hostname: String, include_html: bool) -> Resul
 
     create_directory(dir)?;
 
-    let mut feed = feed::build_atom_feed(&hostname);
+    let feed_file = filename
+        .file_name()
+        .expect("Feed path should have a file name")
+        .to_str()
+        .expect("Feed path should be printable.");
 
-    let mut reader = DataDirectoryMessageReader::new((&Path::new("data")).to_path_buf());
+    let mut feed = feed::build_atom_feed(hostname, feed_file);
 
-    for msg in reader.read_rfc822_messages() {
+    let mut reader = DataDirectoryMessageReader::new(Path::new("data").to_path_buf());
+
+    let max_items_in_feed = max_items_in_feed.unwrap_or_else(|| usize::MAX);
+
+    for msg in reader.read_rfc822_messages().take(max_items_in_feed) {
         let parsed = msg.get_parsed().expect("A parsed messsage.");
 
         let date = parsed.date().ok_or(format!(
@@ -82,15 +97,12 @@ fn build_feed(filename: &PathBuf, hostname: String, include_html: bool) -> Resul
             msg.get_uid()
         ))?;
 
-        let subject = match parsed.subject() {
-            Some(subject) => subject,
-            None => "No subject",
-        };
+        let subject = parsed.subject().unwrap_or("No subject");
 
         println!(
             "Processing message {} from {} with subject {}",
             msg.get_uid(),
-            date.to_string(),
+            date,
             subject
         );
 
@@ -98,26 +110,23 @@ fn build_feed(filename: &PathBuf, hostname: String, include_html: bool) -> Resul
         let processed_html = process_html(&html_body).expect("Could not process the HTML");
 
         if include_html {
-            let path : PathBuf = [dir, Path::new(&get_path(&parsed, &msg))].iter().collect();
-            write_file(&path, processed_html.as_bytes())?;
+            let path: PathBuf = [dir, Path::new(&get_path(&parsed, &msg))].iter().collect();
+            write_file(path, processed_html.as_bytes())?;
         }
 
-        feed::add_entry_to_feed(&mut feed, &msg, &processed_html, &hostname, include_html);
+        feed::add_entry_to_feed(&mut feed, &msg, &processed_html, hostname, include_html);
     }
 
-    if feed.entries.len() > 0 {
+    if !feed.entries.is_empty() {
         feed.set_updated(Utc::now());
         println!("Writing feed to {}", filename.display());
 
-        // TODO: Ugly hack because atom_syndication crate does not support style sheets.
-        let feed_str = feed.to_string().as_str().replace(">\n<feed", ">\n<?xml-stylesheet href=\"feed.xsl\" type=\"text/xsl\"?>\n<feed");
-        let _ = write_file(filename, feed_str)?;
-        let _ = write_file(dir.join("feed.xsl"), FEED_STYLESHEET)?;
-
-        // Another ugly hack, but I don't know how to do this better...
-        let file_name = format!("{:?}", filename.file_name().unwrap()).replace('"', "");
-        write_file(dir.join("index.html"), INDEX_HTML.replace("{FEED}", file_name.as_str()))?;
-
+        feed::write_feed(feed, open_file(filename).unwrap())?;
+        write_file(dir.join("feed.xsl"), FEED_STYLESHEET)?;
+        write_file(
+            dir.join("index.html"),
+            INDEX_HTML.replace("{FEED}", feed_file),
+        )?;
     }
 
     println!("Finished building the feed.");
@@ -134,14 +143,9 @@ fn fetch_from_imap(
 ) -> Result<(), Box<dyn Error>> {
     create_directory(data_directory)?;
 
-    print!("Getting mail from {} for mailbox {}", server, username);
+    println!("Getting mail from {server} for mailbox {username}");
 
-    let mut reader = ImapReader::new(
-        String::from(server),
-        port,
-        String::from(username),
-        String::from(password),
-    );
+    let mut reader = ImapReader::new(server, port, username, password);
 
     for msg in reader.read_rfc822_messages() {
         let parsed = msg.get_parsed().ok_or(format!(
@@ -154,25 +158,19 @@ fn fetch_from_imap(
             msg.get_uid()
         ))?;
 
-        let subject = match parsed.subject() {
-            Some(subject) => subject,
-            None => "No subject",
-        };
+        let subject = parsed.subject().unwrap_or("No subject");
 
         println!(
             "Processing message {} from {} with subject {}",
             msg.get_uid(),
-            date.to_string(),
+            date,
             subject
         );
 
         let path = get_path(&parsed, &msg);
-        let html_path: PathBuf = [
-            Path::new(data_directory),
-            Path::new(&format!("{}.eml", path)),
-        ]
-        .iter()
-        .collect();
+        let html_path: PathBuf = [Path::new(data_directory), Path::new(&format!("{path}.eml"))]
+            .iter()
+            .collect();
 
         println!("Storing to {}", &html_path.display());
 
@@ -182,8 +180,6 @@ fn fetch_from_imap(
     Ok(())
 }
 
-
-
 fn get_path(parsed: &ParsedMessage, msg: &Message) -> String {
     let date = parsed.date().expect("Could not extract date");
     let date_str = format!(
@@ -192,26 +188,32 @@ fn get_path(parsed: &ParsedMessage, msg: &Message) -> String {
     );
 
     let hash = base16ct::lower::encode_string(&Sha256::digest(
-        &parsed.body_html(0).expect("Expected a body").as_bytes(),
+        parsed.body_html(0).expect("Expected a body").as_bytes(),
     ));
 
-    let uid: i32 = msg.get_uid()
+    let uid: i32 = msg
+        .get_uid()
         .parse()
-        .expect(&format!("Could not convert message id {} to an i32.", msg.get_uid()));
+        .unwrap_or_else(|_| panic!("Could not convert message id {} to an i32.", msg.get_uid()));
 
-    format!("{:05}_{}_{}.html", uid, date_str, &hash).to_owned()
+    format!("{:05}_{}_{}.html", uid, date_str, &hash)
 }
 
 fn process_html(input: &str) -> Result<String, ()> {
     Ok(input.replace("src", "data-source"))
 }
 
-fn write_file<P: Into<PathBuf>, D: AsRef<[u8]>>(html_path: P, data: D) -> Result<(), std::io::Error> {
-    let path : PathBuf = html_path.into();
+fn open_file<P: Into<PathBuf>>(path: P) -> std::io::Result<std::fs::File> {
     OpenOptions::new()
         .write(true)
+        .truncate(true)
         .create(true)
-        .open(&path)
-        .expect(format!("Could not open file '{}' for writing", &path.display()).as_str())
+        .open(path.into())
+}
+
+fn write_file<P: Into<PathBuf>, D: AsRef<[u8]>>(path: P, data: D) -> Result<(), std::io::Error> {
+    let path: PathBuf = path.into();
+    open_file(path.clone())
+        .unwrap_or_else(|_| panic!("Could not open file '{}' for writing", &path.display()))
         .write_all(data.as_ref())
-}
+}

src/message.rs

@@ -21,4 +21,4 @@ impl Message {
     pub fn get_data(&self) -> &Vec<u8> {
         &self.data
     }
-}
+}

src/message_reader.rs

@@ -1,167 +1,13 @@
-use std::{
-    collections::HashMap,
-    error::Error,
-    fs::{read_dir, DirEntry},
-    net::TcpStream,
-    path::PathBuf,
-    vec::IntoIter,
-};
+use std::vec::IntoIter;
 
-use imap::Session;
-use rustls_connector::RustlsConnector;
+pub(crate) mod data_directory;
+pub(crate) mod imap;
+
+pub(crate) use data_directory::DataDirectoryMessageReader;
+pub(crate) use crate::message_reader::imap::ImapReader;
 
 use crate::Message;
 
 pub(crate) trait EmailReader {
     fn read_rfc822_messages(&mut self) -> Box<IntoIter<Message>>;
 }
-
-pub(crate) struct DataDirectoryMessageReader {
-    path: PathBuf,
-}
-
-impl DataDirectoryMessageReader {
-    pub fn new(path: PathBuf) -> Self {
-        DataDirectoryMessageReader { path }
-    }
-}
-
-impl EmailReader for DataDirectoryMessageReader {
-    fn read_rfc822_messages(&mut self) -> Box<IntoIter<Message>> {
-        println!("Reading files in {}", &self.path.display());
-        let reader = match read_dir(&self.path) {
-            Ok(r) => r,
-            Err(e) => {
-                dbg!(e);
-                return Box::new(Vec::new().into_iter());
-            }
-        };
-
-        let items = reader
-            .filter(|i| i.is_ok())
-            .map(|i| i.unwrap() as DirEntry)
-            .filter(|d| match d.path().extension() {
-                Some(ext) => ext == "eml",
-                None => false,
-            })
-            .map(|i| {
-                let uid = i
-                    .path()
-                    .file_stem()
-                    .unwrap()
-                    .to_owned()
-                    .into_string()
-                    .expect("Could not convert filename to string.")
-                    .split("_")
-                    .collect::<Vec<&str>>()[0]
-                    .trim_start_matches("0")
-                    .to_string();
-
-                if let Ok(data) = std::fs::read(i.path()) {
-                    Some((uid, data))
-                } else {
-                    None
-                }
-            })
-            .filter(|i| i.is_some())
-            .map(|i| i.unwrap())
-            .map(|i| Message::new(i.0, i.1));
-
-        let iter = items.collect::<Vec<Message>>().into_iter();
-
-        Box::new(iter)
-    }
-}
-
-pub struct ImapReader {
-    host: String,
-    port: u16,
-    username: String,
-    password: String,
-}
-
-impl ImapReader {
-    pub fn new(host: String, port: u16, username: String, password: String) -> Self {
-        ImapReader {
-            host,
-            port,
-            username,
-            password,
-        }
-    }
-
-    fn connect(&self) -> Result<HashMap<String, Vec<u8>>, Box<dyn Error>> {
-        let mut session = self.open_session()?;
-
-        session.examine("INBOX")?;
-
-        let items = match session.uid_search("ALL") {
-            Ok(i) => i,
-            Err(e) => return Err(Box::new(e)),
-        };
-
-        let mut msgs = HashMap::<String, Vec<u8>>::with_capacity(items.len());
-
-        //println!("# of messages: {}", &items.len());
-
-        for item in items {
-            let msg = session.uid_fetch(&item.to_string(), "(BODY.PEEK[] UID)")?;
-
-            let message = if let Some(m) = msg.iter().next() {
-                m
-            } else {
-                continue;
-            };
-
-            let body = message.body().expect("Message did not have a body.");
-
-            msgs.insert(item.to_string(), body.to_owned());
-        }
-
-        session.logout().expect("Could not log out");
-
-        Ok(msgs)
-    }
-
-    fn open_session(
-        &self,
-    ) -> Result<
-        Session<
-            rustls_connector::rustls::StreamOwned<
-                rustls_connector::rustls::ClientConnection,
-                TcpStream,
-            >,
-        >,
-        Box<dyn Error + 'static>,
-    > {
-        let stream = TcpStream::connect((self.host.as_ref(), self.port))?;
-        let tls = RustlsConnector::new_with_webpki_roots_certs();
-        let tls_stream = tls.connect(&self.host, stream)?;
-
-        let client = imap::Client::new(tls_stream);
-
-        Ok(client
-            .login(&self.username, &self.password)
-            .map_err(|e| e.0)?)
-    }
-}
-
-impl EmailReader for ImapReader {
-    fn read_rfc822_messages(&mut self) -> Box<IntoIter<Message>> {
-        let msgs = match self.connect() {
-            Ok(m) => m,
-            Err(e) => {
-                dbg!(e);
-                return Box::new(Vec::new().into_iter());
-            }
-        };
-
-        let items = msgs
-            .iter()
-            .map(|i| Message::new(i.0.to_owned(), i.1.to_owned()));
-
-        let iter = items.collect::<Vec<Message>>().into_iter();
-
-        Box::new(iter)
-    }
-}

src/message_reader/data_directory.rs

@@ -0,0 +1,64 @@
+use crate::message::Message;
+use crate::message_reader::EmailReader;
+use std::{
+    fs::{read_dir, DirEntry},
+    path::PathBuf,
+    vec::IntoIter,
+};
+
+pub(crate) struct DataDirectoryMessageReader {
+    path: PathBuf,
+}
+
+impl DataDirectoryMessageReader {
+    pub fn new(path: PathBuf) -> Self {
+        DataDirectoryMessageReader { path }
+    }
+}
+
+impl EmailReader for DataDirectoryMessageReader {
+    fn read_rfc822_messages(&mut self) -> Box<IntoIter<Message>> {
+        println!("Reading files in {}", &self.path.display());
+        let reader = match read_dir(&self.path) {
+            Ok(r) => r,
+            Err(e) => {
+                dbg!(e);
+                return Box::new(Vec::new().into_iter());
+            }
+        };
+
+        let items = reader
+            .filter(|i| i.is_ok())
+            .map(|i| i.unwrap() as DirEntry)
+            .filter(|d| match d.path().extension() {
+                Some(ext) => ext == "eml",
+                None => false,
+            })
+            .filter_map(|i| {
+                let uid = i
+                    .path()
+                    .file_stem()
+                    .unwrap()
+                    .to_owned()
+                    .into_string()
+                    .expect("Could not convert filename to string.")
+                    .split('_')
+                    .collect::<Vec<&str>>()[0]
+                    .trim_start_matches('0')
+                    .to_string();
+
+                if let Ok(data) = std::fs::read(i.path()) {
+                    Some((uid, data))
+                } else {
+                    None
+                }
+            })
+            .map(|i| Message::new(i.0, i.1));
+
+        let mut items = items.collect::<Vec<Message>>();
+        items.sort_by_key(|m| m.get_parsed().unwrap().date().unwrap().to_owned());
+        let items = items.into_iter().rev().collect::<Vec<_>>();
+
+        Box::new(items.into_iter())
+    }
+}

src/message_reader/imap.rs

@@ -0,0 +1,114 @@
+use std::io::Write;
+use std::{collections::HashMap, error::Error, net::TcpStream, vec::IntoIter};
+
+use imap::Session;
+use rustls_connector::RustlsConnector;
+
+use crate::message::Message;
+use crate::message_reader::EmailReader;
+
+pub struct ImapReader {
+    host: String,
+    port: u16,
+    username: String,
+    password: String,
+}
+
+impl ImapReader {
+    pub fn new(host: String, port: u16, username: String, password: String) -> Self {
+        ImapReader {
+            host,
+            port,
+            username,
+            password,
+        }
+    }
+
+    fn connect(&self) -> Result<HashMap<String, Vec<u8>>, Box<dyn Error>> {
+        let mut session = self.open_session()?;
+
+        session.examine("INBOX")?;
+
+        let mut items = match session.uid_search("ALL") {
+            Ok(i) => i.into_iter().collect::<Vec<_>>(),
+            Err(e) => return Err(Box::new(e)),
+        };
+
+        items.sort();
+
+        let mut msgs = HashMap::with_capacity(items.len());
+
+        println!("Available messages: {}", &items.len());
+
+        let sequence_sets = items.chunks(100);
+        for set in sequence_sets {
+            let sequence_set = set
+                .iter()
+                .map(|x| x.to_string())
+                .collect::<Vec<_>>()
+                .join(",");
+
+            print!("Fetching {} messages...", set.len());
+            let _ = std::io::stdout().flush();
+
+            let fetched_msgs = session.uid_fetch(sequence_set, "(BODY.PEEK[] UID)")?;
+
+            for message in fetched_msgs.iter() {
+                let uid = message.uid.expect("Message did not have a UID").to_string();
+                let body = message.body().expect("Message did not have a body.");
+
+                msgs.insert(uid, body.to_owned());
+            }
+
+            println!(" done");
+        }
+
+        session.logout().expect("Could not log out");
+
+        Ok(msgs)
+    }
+
+    fn open_session(
+        &self,
+    ) -> Result<
+        Session<
+            rustls_connector::rustls::StreamOwned<
+                rustls_connector::rustls::ClientConnection,
+                TcpStream,
+            >,
+        >,
+        Box<dyn Error + 'static>,
+    > {
+        let stream = TcpStream::connect((self.host.as_ref(), self.port))?;
+        let tls = RustlsConnector::new_with_webpki_roots_certs();
+        let tls_stream = tls.connect(&self.host, stream)?;
+
+        let client = imap::Client::new(tls_stream);
+
+        Ok(client
+            .login(&self.username, &self.password)
+            .map_err(|e| e.0)?)
+    }
+}
+
+impl EmailReader for ImapReader {
+    fn read_rfc822_messages(&mut self) -> Box<IntoIter<Message>> {
+        let messages = match self.connect() {
+            Ok(m) => m,
+            Err(e) => {
+                dbg!(e);
+                return Box::new(Vec::new().into_iter());
+            }
+        };
+
+        let mut items = messages
+            .iter()
+            .map(|i| Message::new(i.0.to_owned(), i.1.to_owned()))
+            .collect::<Vec<_>>();
+
+        items.sort_by_key(|m| m.get_parsed().unwrap().date().unwrap().to_owned());
+        let items = items.into_iter().rev().collect::<Vec<_>>();
+
+        Box::new(items.into_iter())
+    }
+}

systemd/newsletter2web.service

@@ -1,7 +1,42 @@
 [Unit]
 Description=Create newsletter feed
 
+After=network-online.target
+Wants=network-online.target
+
 [Service]
 Type=oneshot
-WorkingDirectory=/home/n2w
+WorkingDirectory=/home/n2w/n2w
 ExecStart=/home/n2w/build-feed.sh
+User=n2w
+
+# Security
+PrivateTmp=yes
+PrivateDevices=yes
+ProtectSystem=strict
+SystemCallFilter=@system-service
+#SystemCallFilter=@basic-io @file-system @network-io mprotect
+CapabilityBoundingSet=
+NoNewPrivileges=yes
+ProtectProc=invisible
+RemoveIPC=yes
+RestrictAddressFamilies=AF_INET AF_INET6
+RestrictNamespaces=yes
+PrivateUsers=yes
+
+# ProtectHostname and ProcSubset=pid cannot go together
+# see: https://github.com/systemd/systemd/pull/22203
+# This is fixed in systemd v251
+#ProtectHostname=yes
+ProtectClock=yes
+ProtectKernalTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+LockPersonality=yes
+MemoryDenyWriteExecute=yes
+RestrictRealtime=yes
+ProcSubset=pid
+UMask=0077
+SystemCallArchitectures=native
+RestrictSUIDSGID=yes