kv-import: 1Password .1pux and CSV import, encrypted export (M5)

New crate kv-import (pure parsing, no store) and kv-ffi/src/import.rs:
- .1pux: accounts -> vaults -> items; Login/Password/Card/Note/Identity mapped, other categories
  as secure notes; unmapped fields appended to notes; attachments and password history reported.
- CSV: 1Password 8 export and generic headers (aliases, BOM, quotes, embedded newlines,
  extra columns into notes).
- Duplicates on (title, username, URL host): skip or add " (imported)".
- Encrypted export KVEX v1: Argon2id (>= account KDF minimum) + XChaCha20-Poly1305,
  authenticated header, padded JSON; restorable through import.
- Caps: 100 MB input, 64 MB decompressed export.data, 10k zip entries, 100k items.
- FFI: VaultCore.import_preview / import_commit / export_encrypted (unlocked only); items are
  sealed, seq-stamped and queued for sync like save_item.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

crates/kv-ffi/Cargo.toml

@@ -20,6 +20,7 @@ kv-crypto.workspace = true
 kv-model.workspace = true
 kv-store.workspace = true
 kv-sync.workspace = true
+kv-import.workspace = true
 serde.workspace = true
 hkdf.workspace = true
 sha2.workspace = true
@@ -33,3 +34,4 @@ uniffi = { workspace = true, features = ["cli", "tokio"] }
 [dev-dependencies]
 kv-sync = { workspace = true, features = ["test-server"] }
 tempfile.workspace = true
+zip = { version = "9", default-features = false, features = ["deflate-flate2"] }

crates/kv-ffi/src/import.rs

@@ -0,0 +1,345 @@
+//! Import (1Password `.1pux`, CSV, Kestrove encrypted export) and encrypted export
+//! (kestrove-spec/docs/09-import-export.md). Parsing lives in `kv-import`; this module only
+//! dedups against the vault and stores items the same way `save_item` does (sealed, seq-stamped,
+//! queued for sync).
+
+use super::*;
+use kv_import::{DuplicatePolicy, Format, ImportError, Parsed};
+
+#[derive(Debug, Clone, uniffi::Enum)]
+pub enum ImportFormat {
+    /// 1Password Unencrypted Export (`.1pux`).
+    OnePux,
+    /// 1Password CSV, or any CSV with a `title,url,username,password,otp,notes`-style header.
+    Csv,
+    /// A file from `export_encrypted`.
+    KestroveEncrypted { passphrase: String },
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, uniffi::Enum)]
+pub enum DuplicateHandling {
+    /// Keep the existing item, drop the imported one.
+    Skip,
+    /// Import it with " (imported)" appended to the title.
+    AddSuffix,
+}
+
+#[derive(Debug, Clone, uniffi::Record)]
+pub struct ImportOptions {
+    pub duplicates: DuplicateHandling,
+}
+
+/// What `import_commit` would do. Contains no secrets.
+#[derive(Debug, Clone, PartialEq, Eq, uniffi::Record)]
+pub struct ImportPreview {
+    pub total: u32,
+    pub logins: u32,
+    pub secure_notes: u32,
+    pub cards: u32,
+    pub identities: u32,
+    /// Items matching an existing item (or an earlier one in the file) on title, username and URL host.
+    pub duplicates: u32,
+    pub attachments_skipped: u32,
+    pub warnings: Vec<String>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, uniffi::Record)]
+pub struct ImportReport {
+    pub imported: u32,
+    pub skipped_duplicates: u32,
+    /// Duplicates imported with the " (imported)" suffix.
+    pub renamed_duplicates: u32,
+    pub attachments_skipped: u32,
+    pub warnings: Vec<String>,
+    pub item_ids: Vec<String>,
+}
+
+impl From<ImportError> for CoreError {
+    fn from(e: ImportError) -> Self {
+        match e {
+            ImportError::BadPassphrase => CoreError::BadCredentials,
+            ImportError::Crypto => CoreError::Internal { msg: e.to_string() },
+            other => CoreError::Invalid { msg: other.to_string() },
+        }
+    }
+}
+
+fn to_format(f: ImportFormat) -> Format {
+    match f {
+        ImportFormat::OnePux => Format::OnePux,
+        ImportFormat::Csv => Format::Csv,
+        // `Format` wipes the passphrase on drop.
+        ImportFormat::KestroveEncrypted { passphrase } => Format::Encrypted { passphrase },
+    }
+}
+
+#[uniffi::export]
+impl VaultCore {
+    /// Parses `data` and reports what would be imported. Needs an unlocked vault (for duplicate checks).
+    pub fn import_preview(&self, data: Vec<u8>, format: ImportFormat) -> Result<ImportPreview> {
+        let (acct, u) = self.unlocked_snapshot()?;
+        let mut parsed = parse(data, format)?;
+        let existing = self.all_items(&acct, &u)?;
+        let mut preview = ImportPreview {
+            total: parsed.items.len() as u32,
+            logins: parsed.count(ItemKind::Login),
+            secure_notes: parsed.count(ItemKind::SecureNote),
+            cards: parsed.count(ItemKind::Card),
+            identities: parsed.count(ItemKind::Identity),
+            duplicates: 0,
+            attachments_skipped: parsed.attachments_skipped,
+            warnings: std::mem::take(&mut parsed.warnings),
+        };
+        let items = std::mem::take(&mut parsed.items);
+        preview.duplicates =
+            kv_import::plan(items, existing.iter().map(kv_import::dedup::key), DuplicatePolicy::Skip).skipped;
+        Ok(preview)
+    }
+
+    /// Imports `data` into the primary vault. Every item is a new item (fresh id, `seq` 1),
+    /// sealed and queued for the next `sync_now`.
+    pub fn import_commit(&self, data: Vec<u8>, format: ImportFormat, options: ImportOptions) -> Result<ImportReport> {
+        let (acct, u) = self.unlocked_snapshot()?;
+        let mut parsed = parse(data, format)?;
+        let existing = self.all_items(&acct, &u)?;
+        let policy = match options.duplicates {
+            DuplicateHandling::Skip => DuplicatePolicy::Skip,
+            DuplicateHandling::AddSuffix => DuplicatePolicy::AddSuffix,
+        };
+        let plan =
+            kv_import::plan(std::mem::take(&mut parsed.items), existing.iter().map(kv_import::dedup::key), policy);
+        let mut item_ids = Vec::with_capacity(plan.to_add.len());
+        for item in plan.to_add {
+            item_ids.push(self.insert_new(&acct, &u, item)?.to_string());
+        }
+        Ok(ImportReport {
+            imported: item_ids.len() as u32,
+            skipped_duplicates: plan.skipped,
+            renamed_duplicates: plan.renamed,
+            attachments_skipped: parsed.attachments_skipped,
+            warnings: std::mem::take(&mut parsed.warnings),
+            item_ids,
+        })
+    }
+
+    /// All items of the primary vault (including passkeys and Android apps), as a passphrase-encrypted
+    /// `.kvex` file (Argon2id + XChaCha20-Poly1305). The passphrase needs at least 10 characters.
+    /// Restore it with `import_commit(…, KestroveEncrypted { passphrase }, …)`.
+    pub fn export_encrypted(&self, passphrase: String) -> Result<Vec<u8>> {
+        let passphrase = Zeroizing::new(passphrase);
+        let (acct, u) = self.unlocked_snapshot()?;
+        let items = self.all_items(&acct, &u)?;
+        Ok(kv_import::export::encrypt(&items, &passphrase, now())?)
+    }
+}
+
+fn parse(data: Vec<u8>, format: ImportFormat) -> Result<Parsed> {
+    let data = Zeroizing::new(data);
+    Ok(kv_import::parse(&data, &to_format(format), now())?)
+}
+
+impl VaultCore {
+    fn all_items(&self, acct: &AccountRecord, u: &Unlocked) -> Result<Vec<Item>> {
+        let mut out = Vec::new();
+        for li in self.store.list(acct.primary_vault)? {
+            if let Some(item) = decrypt(acct, &u.vault_key, &li.record)? {
+                out.push(item);
+            }
+        }
+        Ok(out)
+    }
+
+    /// The `save_item` path for an item that does not exist yet: seq-stamp, seal, queue for push.
+    /// Unlike `save_item` it keeps every model field (created_at, passkeys, Android apps).
+    fn insert_new(&self, acct: &AccountRecord, u: &Unlocked, mut model: Item) -> Result<Uuid> {
+        let vault = acct.primary_vault;
+        let id = Uuid::new_v4();
+        model.id = id;
+        model.modified_at = model.modified_at.max(model.created_at);
+        model.seq = kv_sync::engine::next_local_version(self.store.as_ref(), acct.account_id, vault, &u.vault_key, id)?;
+        let json = Zeroizing::new(serde_json::to_vec(&model).map_err(internal)?);
+        let sealed = seal_item(acct.account_id, vault, id, &u.vault_key, &json)?;
+        let device = self
+            .lock_inner()
+            .device
+            .as_ref()
+            .map_or(Uuid::nil(), |d| kv_sync::DeviceSigner::device_id(d.signer.as_ref()));
+        self.store.put_local(
+            vault,
+            &ItemRecord {
+                item_id: id,
+                revision: 0,
+                deleted: false,
+                wrapped_item_key: Some(sealed.wrapped_item_key),
+                payload: Some(sealed.payload),
+                updated_by: device,
+                updated_at: String::new(),
+            },
+        )?;
+        Ok(id)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::io::{Cursor, Write};
+
+    use kv_sync::{
+        signing::SoftwareSigner,
+        test_server::{TestServer, SETUP_TOKEN},
+    };
+
+    use super::*;
+
+    const PW: &str = "a long master password";
+    const CSV: &[u8] = include_bytes!("../../kv-import/tests/fixtures/1password8.csv");
+    const EXPORT_DATA: &[u8] = include_bytes!("../../kv-import/tests/fixtures/1pux-export.data.json");
+
+    fn onepux() -> Vec<u8> {
+        let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));
+        let opts = zip::write::SimpleFileOptions::default();
+        zip.start_file("export.data", opts).unwrap();
+        zip.write_all(EXPORT_DATA).unwrap();
+        zip.start_file("files/doc0000000000000000000000a__policy.pdf", opts).unwrap();
+        zip.write_all(b"%PDF synthetic").unwrap();
+        zip.finish().unwrap().into_inner()
+    }
+
+    fn skip() -> ImportOptions {
+        ImportOptions { duplicates: DuplicateHandling::Skip }
+    }
+
+    struct Signer(SoftwareSigner);
+
+    impl DeviceSigner for Signer {
+        fn device_id(&self) -> String {
+            kv_sync::DeviceSigner::device_id(&self.0).to_string()
+        }
+        fn public_key(&self) -> Vec<u8> {
+            kv_sync::DeviceSigner::public_key(&self.0)
+        }
+        fn sign(&self, message: Vec<u8>) -> Result<Vec<u8>> {
+            kv_sync::DeviceSigner::sign(&self.0, &message).map_err(internal)
+        }
+    }
+
+    fn connect(core: &VaultCore, server: &TestServer) {
+        core.configure_server(server.url(), vec![]).unwrap();
+        core.set_device_signer(
+            Arc::new(Signer(SoftwareSigner::generate(Uuid::new_v4()))),
+            "t".into(),
+            PlatformDto::Linux,
+        )
+        .unwrap();
+    }
+
+    #[test]
+    fn needs_unlocked_vault() {
+        let core = VaultCore::new_in_memory();
+        let sk = core.create_account(PW.into()).unwrap();
+        core.lock();
+        assert!(matches!(core.import_preview(CSV.to_vec(), ImportFormat::Csv), Err(CoreError::Locked)));
+        assert!(matches!(core.import_commit(CSV.to_vec(), ImportFormat::Csv, skip()), Err(CoreError::Locked)));
+        assert!(matches!(core.export_encrypted("export passphrase".into()), Err(CoreError::Locked)));
+        core.unlock(PW.into(), sk).unwrap();
+        assert!(matches!(
+            core.import_preview(b"garbage".to_vec(), ImportFormat::OnePux),
+            Err(CoreError::Invalid { .. })
+        ));
+    }
+
+    #[test]
+    fn csv_preview_commit_and_duplicates() {
+        let core = VaultCore::new_in_memory();
+        core.create_account(PW.into()).unwrap();
+        let p = core.import_preview(CSV.to_vec(), ImportFormat::Csv).unwrap();
+        assert_eq!((p.total, p.logins, p.duplicates), (3, 3, 0));
+        assert!(core.list_items().unwrap().is_empty(), "preview stores nothing");
+
+        let r = core.import_commit(CSV.to_vec(), ImportFormat::Csv, skip()).unwrap();
+        assert_eq!((r.imported, r.skipped_duplicates, r.item_ids.len()), (3, 0, 3));
+        let git = core.list_items().unwrap().into_iter().find(|i| i.title == "Example Git").unwrap();
+        assert!(git.has_totp && git.favorite);
+
+        assert_eq!(core.import_preview(CSV.to_vec(), ImportFormat::Csv).unwrap().duplicates, 3);
+        let r = core.import_commit(CSV.to_vec(), ImportFormat::Csv, skip()).unwrap();
+        assert_eq!((r.imported, r.skipped_duplicates), (0, 3));
+        let r = core
+            .import_commit(CSV.to_vec(), ImportFormat::Csv, ImportOptions { duplicates: DuplicateHandling::AddSuffix })
+            .unwrap();
+        assert_eq!((r.imported, r.renamed_duplicates), (3, 3));
+        assert_eq!(core.list_items().unwrap().iter().filter(|i| i.title.ends_with(" (imported)")).count(), 3);
+    }
+
+    #[test]
+    fn onepux_preview_and_commit() {
+        let core = VaultCore::new_in_memory();
+        core.create_account(PW.into()).unwrap();
+        let p = core.import_preview(onepux(), ImportFormat::OnePux).unwrap();
+        assert_eq!((p.total, p.logins, p.secure_notes, p.cards, p.identities), (8, 4, 2, 1, 1));
+        assert_eq!(p.attachments_skipped, 1);
+        assert!(p.warnings.iter().any(|w| w.contains("policy.pdf")));
+
+        let r = core.import_commit(onepux(), ImportFormat::OnePux, skip()).unwrap();
+        assert_eq!(r.imported, 8);
+        let summary = core.list_items().unwrap();
+        let card = summary.iter().find(|i| i.kind == ItemKindDto::Card).unwrap();
+        let card = core.get_item(card.id.clone()).unwrap();
+        assert!(card.notes.unwrap().contains("number: 4111111111111111"));
+        let git = summary.iter().find(|i| i.title == "Example Git").unwrap();
+        let git = core.get_item(git.id.clone()).unwrap();
+        assert_eq!(git.password.as_deref(), Some("correct-horse-battery-staple"));
+        assert_eq!(git.tags, vec!["dev", "work", "Personal"]);
+        // Imported items are ordinary items: editable with seq advancing.
+        let id = core.save_item(git).unwrap();
+        assert_eq!(core.item_history(id).unwrap().len(), 1);
+    }
+
+    #[test]
+    fn encrypted_export_round_trip() {
+        let a = VaultCore::new_in_memory();
+        a.create_account(PW.into()).unwrap();
+        a.import_commit(onepux(), ImportFormat::OnePux, skip()).unwrap();
+        assert!(matches!(a.export_encrypted("short".into()), Err(CoreError::Invalid { .. })));
+        let blob = a.export_encrypted("export passphrase 1".into()).unwrap();
+
+        let b = VaultCore::new_in_memory();
+        b.create_account(PW.into()).unwrap();
+        let wrong = ImportFormat::KestroveEncrypted { passphrase: "not the passphrase".into() };
+        assert!(matches!(b.import_preview(blob.clone(), wrong), Err(CoreError::BadCredentials)));
+        let fmt = ImportFormat::KestroveEncrypted { passphrase: "export passphrase 1".into() };
+        assert_eq!(b.import_preview(blob.clone(), fmt.clone()).unwrap().total, 8);
+        assert_eq!(b.import_commit(blob.clone(), fmt.clone(), skip()).unwrap().imported, 8);
+
+        let full = |c: &VaultCore| {
+            let mut v: Vec<ItemDto> = c.list_items().unwrap().into_iter().map(|s| c.get_item(s.id).unwrap()).collect();
+            v.sort_by(|x, y| x.title.cmp(&y.title));
+            v.into_iter()
+                .map(|d| (d.title, d.kind, d.username, d.password, d.urls, d.totp, d.notes, d.tags, d.favorite))
+                .collect::<Vec<_>>()
+        };
+        assert_eq!(full(&a), full(&b));
+        // Restoring into the same vault again only finds duplicates.
+        assert_eq!(b.import_commit(blob, fmt, skip()).unwrap().skipped_duplicates, 8);
+    }
+
+    #[tokio::test]
+    async fn imported_items_sync() {
+        let server = TestServer::start().await;
+        let dir = tempfile::tempdir().unwrap();
+        let a = VaultCore::open(dir.path().join("a.sqlite").to_string_lossy().into_owned()).unwrap();
+        let sk = a.create_account(PW.into()).unwrap();
+        connect(&a, &server);
+        a.register_account(SETUP_TOKEN.into()).await.unwrap();
+        let r = a.import_commit(onepux(), ImportFormat::OnePux, skip()).unwrap();
+        assert_eq!(a.sync_now().await.unwrap().pushed, r.imported);
+
+        let b = VaultCore::open(dir.path().join("b.sqlite").to_string_lossy().into_owned()).unwrap();
+        connect(&b, &server);
+        assert_eq!(b.login(PW.into(), sk, None).await.unwrap(), LoginResult::Ok);
+        b.sync_now().await.unwrap();
+        assert_eq!(b.list_items().unwrap().len(), 8);
+        let id = r.item_ids[0].clone();
+        assert_eq!(b.get_item(id.clone()).unwrap().title, a.get_item(id).unwrap().title);
+    }
+}

crates/kv-ffi/src/lib.rs

@@ -32,6 +32,9 @@ use zeroize::Zeroizing;
 
 uniffi::setup_scaffolding!();
 
+mod import;
+pub use import::{DuplicateHandling, ImportFormat, ImportOptions, ImportPreview, ImportReport};
+
 #[derive(Debug, thiserror::Error, uniffi::Error)]
 pub enum CoreError {
     #[error("vault is locked")]