Sync upstream main (647d00f6)

README.md

@@ -64,7 +64,7 @@ This crate exposes a few features for controlling dependency usage:
 
 ```rust
 use rustic_backend::BackendOptions;
-use rustic_core::{ConfigOptions, KeyOptions, Repository, RepositoryOptions};
+use rustic_core::{Credentials, ConfigOptions, KeyOptions, Repository, RepositoryOptions};
 use simplelog::{Config, LevelFilter, SimpleLogger};
 use std::error::Error;
 
@@ -78,10 +78,11 @@ fn main() -> Result<(), Box<dyn Error>> {
         .to_backends()?;
 
     // Init repository
-    let repo_opts = RepositoryOptions::default().password("test");
+    let creds = Credentials::Password("test".into());
+    let repo_opts = RepositoryOptions::default();
     let key_opts = KeyOptions::default();
     let config_opts = ConfigOptions::default();
-    let _repo = Repository::new(&repo_opts, &backends)?.init(&key_opts, &config_opts)?;
+    let _repo = Repository::new(&repo_opts, &backends)?.init(&creds, &key_opts, &config_opts)?;
 
     // -> use _repo for any operation on an open repository
     Ok(())
@@ -92,7 +93,7 @@ fn main() -> Result<(), Box<dyn Error>> {
 
 ```rust
 use rustic_backend::BackendOptions;
-use rustic_core::{BackupOptions, PathList, Repository, RepositoryOptions, SnapshotOptions};
+use rustic_core::{BackupOptions, Credentials, PathList, Repository, RepositoryOptions, SnapshotOptions};
 use simplelog::{Config, LevelFilter, SimpleLogger};
 use std::error::Error;
 
@@ -107,10 +108,11 @@ fn main() -> Result<(), Box<dyn Error>> {
         .to_backends()?;
 
     // Open repository
-    let repo_opts = RepositoryOptions::default().password("test");
+    let creds = Credentials::Password("test".into());
+    let repo_opts = RepositoryOptions::default();
 
     let repo = Repository::new(&repo_opts, &backends)?
-        .open()?
+        .open(&creds)?
         .to_indexed_ids()?;
 
     let backup_opts = BackupOptions::default();
@@ -131,7 +133,7 @@ fn main() -> Result<(), Box<dyn Error>> {
 
 ```rust
 use rustic_backend::BackendOptions;
-use rustic_core::{LocalDestination, LsOptions, Repository, RepositoryOptions, RestoreOptions};
+use rustic_core::{Credentials, LocalDestination, LsOptions, Repository, RepositoryOptions, RestoreOptions};
 use simplelog::{Config, LevelFilter, SimpleLogger};
 use std::error::Error;
 
@@ -145,9 +147,10 @@ fn main() -> Result<(), Box<dyn Error>> {
         .to_backends()?;
 
     // Open repository
-    let repo_opts = RepositoryOptions::default().password("test");
+    let creds = Credentials::Password("test".into());
+    let repo_opts = RepositoryOptions::default();
     let repo = Repository::new(&repo_opts, &backends)?
-        .open()?
+        .open(&creds)?
         .to_indexed()?;
 
     // use latest snapshot without filtering snapshots
@@ -175,7 +178,7 @@ fn main() -> Result<(), Box<dyn Error>> {
 
 ```rust
 use rustic_backend::BackendOptions;
-use rustic_core::{CheckOptions, Repository, RepositoryOptions};
+use rustic_core::{CheckOptions, Credentials, Repository, RepositoryOptions};
 use simplelog::{Config, LevelFilter, SimpleLogger};
 use std::error::Error;
 
@@ -189,8 +192,9 @@ fn main() -> Result<(), Box<dyn Error>> {
         .to_backends()?;
 
     // Open repository
-    let repo_opts = RepositoryOptions::default().password("test");
-    let repo = Repository::new(&repo_opts, &backends)?.open()?;
+    let creds = Credentials::Password("test".into());
+    let repo_opts = RepositoryOptions::default();
+    let repo = Repository::new(&repo_opts, &backends)?.open(&creds)?;
 
     // Check repository with standard options but omitting cache checks
     let opts = CheckOptions::default().trust_cache(true);

crates/backend/Cargo.toml

@@ -35,13 +35,22 @@ cli = ["merge", "clap"]
 merge = ["dep:conflate"]
 clap = ["dep:clap"]
 opendal = [
+  "reqwest",
   "dep:opendal",
+  "dep:opendal-http-transport-reqwest",
   "dep:rayon",
   "dep:tokio",
-  "tokio/rt-multi-thread",
   "dep:typed-path",
+  "tokio/rt-multi-thread",
+]
+reqwest = ["dep:reqwest"]
+rest = [
+  "reqwest",
+  "dep:backon",
+  "dep:futures-util",
+  "dep:tokio",
+  "tokio/rt-multi-thread",
 ]
-rest = ["dep:reqwest", "dep:backon"]
 rclone = ["rest", "dep:rand", "dep:semver"]
 
 [dependencies]
@@ -63,7 +72,10 @@ strum = { version = "0.28.0", features = ["derive"] }
 
 # general / backend choosing
 hex = { version = "0.4.3", features = ["serde"] }
+rayon = { version = "1.11.0", optional = true }
 serde = { version = "1.0.228" }
+tokio = { version = "1.49.0", optional = true, default-features = false }
+typed-path = { version = "0.12.2", optional = true }
 url = "2.5.8"
 
 # cli support
@@ -74,28 +86,29 @@ conflate = { version = "0.3.3", optional = true }
 aho-corasick = { workspace = true }
 walkdir = "2.5.0"
 
+# reqwest
+reqwest = { version = "0.13.2", default-features = false, features = ["json", "rustls", "stream"], optional = true }
+
 # rest backend
 backon = { version = "1.6.0", optional = true }
-reqwest = { version = "0.13.2", default-features = false, features = ["json", "rustls", "stream", "blocking"], optional = true }
+futures-util = { version = "0.3.34", default-features = false, optional = true }
 
 # rclone backend
 rand = { version = "0.10.0", optional = true }
 semver = { version = "1.0.27", optional = true }
 
 # opendal backend
 bytesize = "2.3.1"
-rayon = { version = "1.11.0", optional = true }
-tokio = { version = "1.49.0", optional = true, default-features = false }
-typed-path = { version = "0.12.2", optional = true }
+opendal-http-transport-reqwest = { version = "0.58.2", optional = true }
 
 [target.'cfg(not(windows))'.dependencies]
 # opendal backend
 # - sftp is not supported on windows, see https://github.com/apache/incubator-opendal/issues/2963
-opendal = { version = "0.58.1", features = ["blocking", "services-b2", "services-ftp", "services-sftp", "services-swift", "services-azblob", "services-azdls", "services-cos", "services-fs", "services-dropbox", "services-gdrive", "services-gcs", "services-ghac", "services-http", "services-ipmfs", "services-memory", "services-obs", "services-onedrive", "services-oss", "services-pcloud", "services-s3", "services-webdav", "services-webhdfs", "services-azfile", "layers-throttle", "services-yandex-disk"], optional = true }
+opendal = { version = "0.58.2", features = ["blocking", "services-b2", "services-ftp", "services-sftp", "services-swift", "services-azblob", "services-azdls", "services-cos", "services-fs", "services-dropbox", "services-gdrive", "services-gcs", "services-ghac", "services-http", "services-ipmfs", "services-memory", "services-obs", "services-onedrive", "services-oss", "services-pcloud", "services-s3", "services-webdav", "services-webhdfs", "services-azfile", "layers-throttle", "services-yandex-disk"], optional = true }
 
 [target.'cfg(windows)'.dependencies]
 # opendal backend
-opendal = { version = "0.58.1", features = ["blocking", "services-b2", "services-ftp", "services-swift", "services-azblob", "services-azdls", "services-cos", "services-fs", "services-dropbox", "services-gdrive", "services-gcs", "services-ghac", "services-http", "services-ipmfs", "services-memory", "services-obs", "services-onedrive", "services-oss", "services-pcloud", "services-s3", "services-webdav", "services-webhdfs", "services-azfile", "layers-throttle", "services-yandex-disk"], optional = true }
+opendal = { version = "0.58.2", features = ["blocking", "services-b2", "services-ftp", "services-swift", "services-azblob", "services-azdls", "services-cos", "services-fs", "services-dropbox", "services-gdrive", "services-gcs", "services-ghac", "services-http", "services-ipmfs", "services-memory", "services-obs", "services-onedrive", "services-oss", "services-pcloud", "services-s3", "services-webdav", "services-webhdfs", "services-azfile", "layers-throttle", "services-yandex-disk"], optional = true }
 
 [dev-dependencies]
 anyhow = { workspace = true }

crates/backend/README.md

@@ -10,6 +10,16 @@
 <a href="https://crates.io/crates/rustic_backend"><img src="https://img.shields.io/crates/d/rustic_backend.svg" /></a>
 <p>
 
+<p align="center">
+Most backends featured by<br />
+<a href="https://opendal.apache.org/">
+<picture>
+<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apache/opendal/main/website/static/img/logo_dark.svg">
+<img src="https://raw.githubusercontent.com/apache/opendal/main/website/static/img/logo.svg" alt="Powered by Apache OpenDAL" height="64" />
+</picture>
+</a>
+</p>
+
 ## News
 
 We temporarily disabled the `service-ftp` feature on the `opendal` backend due

crates/backend/src/lib.rs

@@ -61,6 +61,9 @@ pub mod local;
 /// Utility functions for the backend.
 pub mod util;
 
+#[cfg(feature = "reqwest")]
+pub(crate) mod reqwest;
+
 /// `OpenDAL` backend for Rustic.
 #[cfg(feature = "opendal")]
 pub mod opendal;

crates/backend/src/local.rs

@@ -522,6 +522,8 @@ impl WriteBackend for LocalBackend {
 
         trace!("writing tpe: {tpe:?}, id: {id}");
         let filename = self.path(tpe, id);
+        let replaces_existing_config =
+            tpe == FileType::Config && self.post_delete_command.is_some() && filename.exists();
 
         let parent = self.base_path(tpe, id);
 
@@ -565,6 +567,14 @@ impl WriteBackend for LocalBackend {
             .ask_report()
         })?;
 
+        // Replacing the config bypasses `remove`, so invoke its hook explicitly.
+        if replaces_existing_config
+            && let Some(command) = &self.post_delete_command
+            && let Err(err) = Self::call_command(tpe, id, &filename, command)
+        {
+            warn!("post-delete: {}", err.display_log());
+        }
+
         if let Some(command) = &self.post_create_command
             && let Err(err) = Self::call_command(tpe, id, &filename, command)
         {
@@ -603,3 +613,63 @@ impl WriteBackend for LocalBackend {
         Ok(())
     }
 }
+
+#[cfg(all(test, unix))]
+mod tests {
+    use std::{
+        env, fs, process,
+        time::{SystemTime, UNIX_EPOCH},
+    };
+
+    use bytes::Bytes;
+    use rustic_core::{FileType, Id, WriteBackend};
+
+    use super::LocalBackend;
+
+    #[test]
+    fn replacing_config_runs_delete_hook_before_create_hook() {
+        let repo = env::temp_dir().join(format!(
+            "rustic-local-backend-test-{}-{}",
+            process::id(),
+            SystemTime::now()
+                .duration_since(UNIX_EPOCH)
+                .unwrap()
+                .as_nanos()
+        ));
+        fs::create_dir(&repo).unwrap();
+        let marker = repo.join("hooks-ran");
+        let options = [
+            (
+                "post-create-command".to_string(),
+                format!("sh -c 'printf create >> {}'", marker.display()),
+            ),
+            (
+                "post-delete-command".to_string(),
+                format!("sh -c 'printf delete >> {}'", marker.display()),
+            ),
+        ];
+        let backend = LocalBackend::new(repo.to_string_lossy(), options).unwrap();
+        backend.create().unwrap();
+
+        backend
+            .write_bytes(
+                FileType::Config,
+                &Id::default(),
+                false,
+                Bytes::from_static(b"first").into(),
+            )
+            .unwrap();
+        backend
+            .write_bytes(
+                FileType::Config,
+                &Id::default(),
+                false,
+                Bytes::from_static(b"second").into(),
+            )
+            .unwrap();
+
+        let hooks_ran = fs::read_to_string(marker).unwrap();
+        fs::remove_dir_all(repo).unwrap();
+        assert_eq!(hooks_ran, "createdeletecreate");
+    }
+}

crates/backend/src/opendal.rs

@@ -11,11 +11,14 @@ use bytes::Bytes;
 use bytesize::ByteSize;
 use log::{error, trace, warn};
 use opendal::{
-    Entry, Metadata,
+    Entry, HttpTransporter, Metadata, OperationContext,
     blocking::{Operator, StdReader},
-    layers::{ConcurrentLimitLayer, LoggingLayer, RetryLayer, ThrottleLayer},
+    layers::{
+        ConcurrentLimitLayer, LoggingLayer, RetryEvent, RetryInterceptor, RetryLayer, ThrottleLayer,
+    },
     options::{ListOptions, ReadOptions},
 };
+use opendal_http_transport_reqwest::ReqwestTransport;
 use rayon::prelude::{IntoParallelIterator, ParallelIterator};
 use tokio::runtime::Runtime;
 use typed_path::UnixPathBuf;
@@ -26,6 +29,8 @@ use rustic_core::{
     repofile::{Node, NodeType},
 };
 
+use crate::reqwest::reqwest_client;
+
 mod constants {
     /// Default number of retries
     pub(super) const DEFAULT_RETRY: usize = 5;
@@ -47,6 +52,26 @@ fn runtime() -> &'static Runtime {
     })
 }
 
+/// Log `OpenDAL` retries as a single line using `Display`, not `Debug`.
+///
+/// `OpenDAL`'s default interceptor formats the error with `{:?}`, which dumps
+/// the full context/source tree across many lines and fights progress/TUI
+/// output.
+#[derive(Clone, Copy, Debug)]
+struct CompactRetryInterceptor;
+
+impl RetryInterceptor for CompactRetryInterceptor {
+    fn intercept(&self, event: RetryEvent<'_>) {
+        warn!(
+            "Error {err} at {duration:?}, retrying {op:?} (attempt {attempt})",
+            err = event.err,
+            duration = event.retry_after,
+            op = event.op,
+            attempt = event.attempt,
+        );
+    }
+}
+
 /// Throttling parameters
 ///
 /// Note: Throttle implements [`FromStr`] to read it from something like "10kiB,10MB"
@@ -144,6 +169,10 @@ impl OpenDALBackend {
             .map(|t| Throttle::from_str(t))
             .transpose()?;
 
+        let client = reqwest_client(&options)?;
+        let http_transport = HttpTransporter::new(ReqwestTransport::new(client));
+        let operation_context = OperationContext::new().with_http_transport(http_transport);
+
         let scheme = path
             .as_ref()
             .split(':')
@@ -159,8 +188,13 @@ impl OpenDALBackend {
                 )
                 .attach_context("path", path.as_ref().to_string())
                 .attach_context("schema", scheme.to_string())
-            })?
-            .layer(RetryLayer::new().with_max_times(max_retries).with_jitter());
+            })?.with_context(operation_context)
+            .layer(
+                RetryLayer::new()
+                    .with_max_times(max_retries)
+                    .with_jitter()
+                    .with_notify(CompactRetryInterceptor),
+            );
 
         if let Some(Throttle { bandwidth, burst }) = throttle {
             operator = operator.layer(ThrottleLayer::new(bandwidth, burst));
@@ -647,6 +681,27 @@ mod tests {
 
         Ok(())
     }
+
+    #[test]
+    fn opendal_error_display_is_single_line() {
+        let err = opendal::Error::new(opendal::ErrorKind::Unexpected, "send http request")
+            .with_operation("read")
+            .with_context("url", "https://example.com/index/abc")
+            .set_source(std::io::Error::other(
+                "connection closed before message completed",
+            ));
+
+        let display = err.to_string();
+        let debug = format!("{err:?}");
+        assert!(
+            !display.contains('\n'),
+            "Display should be one line, got: {display:?}"
+        );
+        assert!(
+            debug.contains("Context:"),
+            "Debug should dump context, got: {debug}"
+        );
+    }
 }
 
 #[derive(Debug)]