diff --git a/Cargo.lock b/Cargo.lock index f793284..3d04d69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aead" version = "0.5.2" @@ -347,6 +353,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + [[package]] name = "chacha20" version = "0.9.1" @@ -485,6 +497,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + [[package]] name = "crunchy" version = "0.2.4" @@ -742,6 +763,16 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1319,6 +1350,7 @@ dependencies = [ "ciborium", "hkdf", "kv-crypto", + "kv-import", "kv-model", "kv-proto", "kv-store", @@ -1333,6 +1365,27 @@ dependencies = [ "uniffi", "uuid", "zeroize", + "zip", +] + +[[package]] +name = "kv-import" +version = "0.1.0" +dependencies = [ + "argon2", + "chacha20poly1305", + "flate2", + "kv-crypto", + "kv-model", + "kv-proto", + "serde", + "serde_json", + "thiserror 2.0.21", + "unicode-normalization", + "url", + "uuid", + "zeroize", + "zip", ] [[package]] @@ -1483,6 +1536,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.4" @@ -1494,6 +1557,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -2090,6 +2165,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "siphasher" version = "0.3.11" @@ -2438,6 +2519,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.1" @@ -3034,6 +3121,20 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "zip" +version = "9.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a954114b47a54df49f83d2722814e86b23d4bb1b54a1b90fb7d92b41619ddc22" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "memchr", + "nix", + "typed-path", +] + [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index 017c395..441a952 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ members = ["crates/*"] version = "0.1.0" edition = "2021" license = "Apache-2.0" -rust-version = "1.82" +rust-version = "1.88" publish = false [workspace.dependencies] @@ -15,6 +15,7 @@ kv-crypto = { path = "crates/kv-crypto" } kv-model = { path = "crates/kv-model" } kv-store = { path = "crates/kv-store" } kv-sync = { path = "crates/kv-sync" } +kv-import = { path = "crates/kv-import" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/kv-ffi/Cargo.toml b/crates/kv-ffi/Cargo.toml index 7bb2385..fb2d959 100644 --- a/crates/kv-ffi/Cargo.toml +++ b/crates/kv-ffi/Cargo.toml @@ -20,6 +20,7 @@ kv-crypto.workspace = true kv-model.workspace = true kv-store.workspace = true kv-sync.workspace = true +kv-import.workspace = true serde.workspace = true hkdf.workspace = true sha2.workspace = true @@ -35,3 +36,4 @@ kv-sync = { workspace = true, features = ["test-server"] } tempfile.workspace = true p256.workspace = true ciborium.workspace = true +zip = { version = "9", default-features = false, features = ["deflate-flate2"] } diff --git a/crates/kv-ffi/src/import.rs b/crates/kv-ffi/src/import.rs new file mode 100644 index 0000000..8fed583 --- /dev/null +++ b/crates/kv-ffi/src/import.rs @@ -0,0 +1,345 @@ +//! Import (1Password `.1pux`, CSV, Kestrove encrypted export) and encrypted export +//! (kestrove-spec/docs/09-import-export.md). Parsing lives in `kv-import`; this module only +//! dedups against the vault and stores items the same way `save_item` does (sealed, seq-stamped, +//! queued for sync). + +use super::*; +use kv_import::{DuplicatePolicy, Format, ImportError, Parsed}; + +#[derive(Debug, Clone, uniffi::Enum)] +pub enum ImportFormat { + /// 1Password Unencrypted Export (`.1pux`). + OnePux, + /// 1Password CSV, or any CSV with a `title,url,username,password,otp,notes`-style header. + Csv, + /// A file from `export_encrypted`. + KestroveEncrypted { passphrase: String }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, uniffi::Enum)] +pub enum DuplicateHandling { + /// Keep the existing item, drop the imported one. + Skip, + /// Import it with " (imported)" appended to the title. + AddSuffix, +} + +#[derive(Debug, Clone, uniffi::Record)] +pub struct ImportOptions { + pub duplicates: DuplicateHandling, +} + +/// What `import_commit` would do. Contains no secrets. +#[derive(Debug, Clone, PartialEq, Eq, uniffi::Record)] +pub struct ImportPreview { + pub total: u32, + pub logins: u32, + pub secure_notes: u32, + pub cards: u32, + pub identities: u32, + /// Items matching an existing item (or an earlier one in the file) on title, username and URL host. + pub duplicates: u32, + pub attachments_skipped: u32, + pub warnings: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, uniffi::Record)] +pub struct ImportReport { + pub imported: u32, + pub skipped_duplicates: u32, + /// Duplicates imported with the " (imported)" suffix. + pub renamed_duplicates: u32, + pub attachments_skipped: u32, + pub warnings: Vec, + pub item_ids: Vec, +} + +impl From for CoreError { + fn from(e: ImportError) -> Self { + match e { + ImportError::BadPassphrase => CoreError::BadCredentials, + ImportError::Crypto => CoreError::Internal { msg: e.to_string() }, + other => CoreError::Invalid { msg: other.to_string() }, + } + } +} + +fn to_format(f: ImportFormat) -> Format { + match f { + ImportFormat::OnePux => Format::OnePux, + ImportFormat::Csv => Format::Csv, + // `Format` wipes the passphrase on drop. + ImportFormat::KestroveEncrypted { passphrase } => Format::Encrypted { passphrase }, + } +} + +#[uniffi::export] +impl VaultCore { + /// Parses `data` and reports what would be imported. Needs an unlocked vault (for duplicate checks). + pub fn import_preview(&self, data: Vec, format: ImportFormat) -> Result { + let (acct, u) = self.unlocked_snapshot()?; + let mut parsed = parse(data, format)?; + let existing = self.all_items(&acct, &u)?; + let mut preview = ImportPreview { + total: parsed.items.len() as u32, + logins: parsed.count(ItemKind::Login), + secure_notes: parsed.count(ItemKind::SecureNote), + cards: parsed.count(ItemKind::Card), + identities: parsed.count(ItemKind::Identity), + duplicates: 0, + attachments_skipped: parsed.attachments_skipped, + warnings: std::mem::take(&mut parsed.warnings), + }; + let items = std::mem::take(&mut parsed.items); + preview.duplicates = + kv_import::plan(items, existing.iter().map(kv_import::dedup::key), DuplicatePolicy::Skip).skipped; + Ok(preview) + } + + /// Imports `data` into the primary vault. Every item is a new item (fresh id, `seq` 1), + /// sealed and queued for the next `sync_now`. + pub fn import_commit(&self, data: Vec, format: ImportFormat, options: ImportOptions) -> Result { + let (acct, u) = self.unlocked_snapshot()?; + let mut parsed = parse(data, format)?; + let existing = self.all_items(&acct, &u)?; + let policy = match options.duplicates { + DuplicateHandling::Skip => DuplicatePolicy::Skip, + DuplicateHandling::AddSuffix => DuplicatePolicy::AddSuffix, + }; + let plan = + kv_import::plan(std::mem::take(&mut parsed.items), existing.iter().map(kv_import::dedup::key), policy); + let mut item_ids = Vec::with_capacity(plan.to_add.len()); + for item in plan.to_add { + item_ids.push(self.insert_new(&acct, &u, item)?.to_string()); + } + Ok(ImportReport { + imported: item_ids.len() as u32, + skipped_duplicates: plan.skipped, + renamed_duplicates: plan.renamed, + attachments_skipped: parsed.attachments_skipped, + warnings: std::mem::take(&mut parsed.warnings), + item_ids, + }) + } + + /// All items of the primary vault (including passkeys and Android apps), as a passphrase-encrypted + /// `.kvex` file (Argon2id + XChaCha20-Poly1305). The passphrase needs at least 10 characters. + /// Restore it with `import_commit(…, KestroveEncrypted { passphrase }, …)`. + pub fn export_encrypted(&self, passphrase: String) -> Result> { + let passphrase = Zeroizing::new(passphrase); + let (acct, u) = self.unlocked_snapshot()?; + let items = self.all_items(&acct, &u)?; + Ok(kv_import::export::encrypt(&items, &passphrase, now())?) + } +} + +fn parse(data: Vec, format: ImportFormat) -> Result { + let data = Zeroizing::new(data); + Ok(kv_import::parse(&data, &to_format(format), now())?) +} + +impl VaultCore { + fn all_items(&self, acct: &AccountRecord, u: &Unlocked) -> Result> { + let mut out = Vec::new(); + for li in self.store.list(acct.primary_vault)? { + if let Some(item) = decrypt(acct, &u.vault_key, &li.record)? { + out.push(item); + } + } + Ok(out) + } + + /// The `save_item` path for an item that does not exist yet: seq-stamp, seal, queue for push. + /// Unlike `save_item` it keeps every model field (created_at, passkeys, Android apps). + fn insert_new(&self, acct: &AccountRecord, u: &Unlocked, mut model: Item) -> Result { + let vault = acct.primary_vault; + let id = Uuid::new_v4(); + model.id = id; + model.modified_at = model.modified_at.max(model.created_at); + model.seq = kv_sync::engine::next_local_version(self.store.as_ref(), acct.account_id, vault, &u.vault_key, id)?; + let json = Zeroizing::new(serde_json::to_vec(&model).map_err(internal)?); + let sealed = seal_item(acct.account_id, vault, id, &u.vault_key, &json)?; + let device = self + .lock_inner() + .device + .as_ref() + .map_or(Uuid::nil(), |d| kv_sync::DeviceSigner::device_id(d.signer.as_ref())); + self.store.put_local( + vault, + &ItemRecord { + item_id: id, + revision: 0, + deleted: false, + wrapped_item_key: Some(sealed.wrapped_item_key), + payload: Some(sealed.payload), + updated_by: device, + updated_at: String::new(), + }, + )?; + Ok(id) + } +} + +#[cfg(test)] +mod tests { + use std::io::{Cursor, Write}; + + use kv_sync::{ + signing::SoftwareSigner, + test_server::{TestServer, SETUP_TOKEN}, + }; + + use super::*; + + const PW: &str = "a long master password"; + const CSV: &[u8] = include_bytes!("../../kv-import/tests/fixtures/1password8.csv"); + const EXPORT_DATA: &[u8] = include_bytes!("../../kv-import/tests/fixtures/1pux-export.data.json"); + + fn onepux() -> Vec { + let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); + let opts = zip::write::SimpleFileOptions::default(); + zip.start_file("export.data", opts).unwrap(); + zip.write_all(EXPORT_DATA).unwrap(); + zip.start_file("files/doc0000000000000000000000a__policy.pdf", opts).unwrap(); + zip.write_all(b"%PDF synthetic").unwrap(); + zip.finish().unwrap().into_inner() + } + + fn skip() -> ImportOptions { + ImportOptions { duplicates: DuplicateHandling::Skip } + } + + struct Signer(SoftwareSigner); + + impl DeviceSigner for Signer { + fn device_id(&self) -> String { + kv_sync::DeviceSigner::device_id(&self.0).to_string() + } + fn public_key(&self) -> Vec { + kv_sync::DeviceSigner::public_key(&self.0) + } + fn sign(&self, message: Vec) -> Result> { + kv_sync::DeviceSigner::sign(&self.0, &message).map_err(internal) + } + } + + fn connect(core: &VaultCore, server: &TestServer) { + core.configure_server(server.url(), vec![]).unwrap(); + core.set_device_signer( + Arc::new(Signer(SoftwareSigner::generate(Uuid::new_v4()))), + "t".into(), + PlatformDto::Linux, + ) + .unwrap(); + } + + #[test] + fn needs_unlocked_vault() { + let core = VaultCore::new_in_memory(); + let sk = core.create_account(PW.into()).unwrap(); + core.lock(); + assert!(matches!(core.import_preview(CSV.to_vec(), ImportFormat::Csv), Err(CoreError::Locked))); + assert!(matches!(core.import_commit(CSV.to_vec(), ImportFormat::Csv, skip()), Err(CoreError::Locked))); + assert!(matches!(core.export_encrypted("export passphrase".into()), Err(CoreError::Locked))); + core.unlock(PW.into(), sk).unwrap(); + assert!(matches!( + core.import_preview(b"garbage".to_vec(), ImportFormat::OnePux), + Err(CoreError::Invalid { .. }) + )); + } + + #[test] + fn csv_preview_commit_and_duplicates() { + let core = VaultCore::new_in_memory(); + core.create_account(PW.into()).unwrap(); + let p = core.import_preview(CSV.to_vec(), ImportFormat::Csv).unwrap(); + assert_eq!((p.total, p.logins, p.duplicates), (3, 3, 0)); + assert!(core.list_items().unwrap().is_empty(), "preview stores nothing"); + + let r = core.import_commit(CSV.to_vec(), ImportFormat::Csv, skip()).unwrap(); + assert_eq!((r.imported, r.skipped_duplicates, r.item_ids.len()), (3, 0, 3)); + let git = core.list_items().unwrap().into_iter().find(|i| i.title == "Example Git").unwrap(); + assert!(git.has_totp && git.favorite); + + assert_eq!(core.import_preview(CSV.to_vec(), ImportFormat::Csv).unwrap().duplicates, 3); + let r = core.import_commit(CSV.to_vec(), ImportFormat::Csv, skip()).unwrap(); + assert_eq!((r.imported, r.skipped_duplicates), (0, 3)); + let r = core + .import_commit(CSV.to_vec(), ImportFormat::Csv, ImportOptions { duplicates: DuplicateHandling::AddSuffix }) + .unwrap(); + assert_eq!((r.imported, r.renamed_duplicates), (3, 3)); + assert_eq!(core.list_items().unwrap().iter().filter(|i| i.title.ends_with(" (imported)")).count(), 3); + } + + #[test] + fn onepux_preview_and_commit() { + let core = VaultCore::new_in_memory(); + core.create_account(PW.into()).unwrap(); + let p = core.import_preview(onepux(), ImportFormat::OnePux).unwrap(); + assert_eq!((p.total, p.logins, p.secure_notes, p.cards, p.identities), (8, 4, 2, 1, 1)); + assert_eq!(p.attachments_skipped, 1); + assert!(p.warnings.iter().any(|w| w.contains("policy.pdf"))); + + let r = core.import_commit(onepux(), ImportFormat::OnePux, skip()).unwrap(); + assert_eq!(r.imported, 8); + let summary = core.list_items().unwrap(); + let card = summary.iter().find(|i| i.kind == ItemKindDto::Card).unwrap(); + let card = core.get_item(card.id.clone()).unwrap(); + assert!(card.notes.unwrap().contains("number: 4111111111111111")); + let git = summary.iter().find(|i| i.title == "Example Git").unwrap(); + let git = core.get_item(git.id.clone()).unwrap(); + assert_eq!(git.password.as_deref(), Some("correct-horse-battery-staple")); + assert_eq!(git.tags, vec!["dev", "work", "Personal"]); + // Imported items are ordinary items: editable with seq advancing. + let id = core.save_item(git).unwrap(); + assert_eq!(core.item_history(id).unwrap().len(), 1); + } + + #[test] + fn encrypted_export_round_trip() { + let a = VaultCore::new_in_memory(); + a.create_account(PW.into()).unwrap(); + a.import_commit(onepux(), ImportFormat::OnePux, skip()).unwrap(); + assert!(matches!(a.export_encrypted("short".into()), Err(CoreError::Invalid { .. }))); + let blob = a.export_encrypted("export passphrase 1".into()).unwrap(); + + let b = VaultCore::new_in_memory(); + b.create_account(PW.into()).unwrap(); + let wrong = ImportFormat::KestroveEncrypted { passphrase: "not the passphrase".into() }; + assert!(matches!(b.import_preview(blob.clone(), wrong), Err(CoreError::BadCredentials))); + let fmt = ImportFormat::KestroveEncrypted { passphrase: "export passphrase 1".into() }; + assert_eq!(b.import_preview(blob.clone(), fmt.clone()).unwrap().total, 8); + assert_eq!(b.import_commit(blob.clone(), fmt.clone(), skip()).unwrap().imported, 8); + + let full = |c: &VaultCore| { + let mut v: Vec = c.list_items().unwrap().into_iter().map(|s| c.get_item(s.id).unwrap()).collect(); + v.sort_by(|x, y| x.title.cmp(&y.title)); + v.into_iter() + .map(|d| (d.title, d.kind, d.username, d.password, d.urls, d.totp, d.notes, d.tags, d.favorite)) + .collect::>() + }; + assert_eq!(full(&a), full(&b)); + // Restoring into the same vault again only finds duplicates. + assert_eq!(b.import_commit(blob, fmt, skip()).unwrap().skipped_duplicates, 8); + } + + #[tokio::test] + async fn imported_items_sync() { + let server = TestServer::start().await; + let dir = tempfile::tempdir().unwrap(); + let a = VaultCore::open(dir.path().join("a.sqlite").to_string_lossy().into_owned()).unwrap(); + let sk = a.create_account(PW.into()).unwrap(); + connect(&a, &server); + a.register_account(SETUP_TOKEN.into()).await.unwrap(); + let r = a.import_commit(onepux(), ImportFormat::OnePux, skip()).unwrap(); + assert_eq!(a.sync_now().await.unwrap().pushed, r.imported); + + let b = VaultCore::open(dir.path().join("b.sqlite").to_string_lossy().into_owned()).unwrap(); + connect(&b, &server); + assert_eq!(b.login(PW.into(), sk, None).await.unwrap(), LoginResult::Ok); + b.sync_now().await.unwrap(); + assert_eq!(b.list_items().unwrap().len(), 8); + let id = r.item_ids[0].clone(); + assert_eq!(b.get_item(id.clone()).unwrap().title, a.get_item(id).unwrap().title); + } +} diff --git a/crates/kv-ffi/src/lib.rs b/crates/kv-ffi/src/lib.rs index fab1645..a87d53c 100644 --- a/crates/kv-ffi/src/lib.rs +++ b/crates/kv-ffi/src/lib.rs @@ -34,6 +34,8 @@ uniffi::setup_scaffolding!(); mod passkeys; pub use passkeys::{kestrove_aaguid, PasskeyAssertion, PasskeyRegistration, PasskeySummary}; +mod import; +pub use import::{DuplicateHandling, ImportFormat, ImportOptions, ImportPreview, ImportReport}; #[derive(Debug, thiserror::Error, uniffi::Error)] pub enum CoreError { diff --git a/crates/kv-import/Cargo.toml b/crates/kv-import/Cargo.toml new file mode 100644 index 0000000..390a4e2 --- /dev/null +++ b/crates/kv-import/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "kv-import" +description = "Kestrove import (1Password .1pux, CSV) and passphrase-encrypted export." +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +kv-proto.workspace = true +kv-crypto.workspace = true +kv-model.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +uuid.workspace = true +zeroize.workspace = true +argon2.workspace = true +chacha20poly1305.workspace = true +unicode-normalization.workspace = true +url.workspace = true +# Reading only; the pure-Rust miniz_oxide backend keeps the build C-free. +zip = { version = "9", default-features = false, features = ["deflate-flate2"] } +# Selects the pure-Rust miniz_oxide backend for zip (no C toolchain needed on any platform). +flate2 = { version = "1", default-features = false, features = ["rust_backend"] } diff --git a/crates/kv-import/src/csv.rs b/crates/kv-import/src/csv.rs new file mode 100644 index 0000000..0ab9038 --- /dev/null +++ b/crates/kv-import/src/csv.rs @@ -0,0 +1,185 @@ +//! CSV import: 1Password's CSV export (`Title,Url,Username,Password,OTPAuth,Favorite,Archived,Tags,Notes`) +//! and any CSV whose header names columns like `title,url,username,password,otp,notes` +//! (case-insensitive; see [`column_for`]). Columns with no field of their own go into the notes. +//! +//! RFC 4180 parsing: quoted fields, `""` escapes, embedded newlines, CRLF or LF, a UTF-8 BOM. + +use kv_model::ItemKind; +use zeroize::Zeroize; + +use crate::{malformed, new_item, set_opt, Extras, ImportError, Limits, Parsed, Result}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Column { + Title, + Url, + Username, + Password, + Otp, + Notes, + Favorite, + Tags, + Archived, + Other, +} + +/// Header aliases (lowercased, non-alphanumerics removed). Covers 1Password 7/8, Bitwarden, Chrome. +fn column_for(header: &str) -> Column { + let h: String = header.chars().filter(|c| c.is_alphanumeric()).flat_map(char::to_lowercase).collect(); + match h.as_str() { + "title" | "name" => Column::Title, + "url" | "urls" | "website" | "loginuri" | "loginurl" | "uri" => Column::Url, + "username" | "loginusername" | "user" | "login" => Column::Username, + "password" | "loginpassword" => Column::Password, + "otp" | "otpauth" | "totp" | "logintotp" | "onetimepassword" => Column::Otp, + "notes" | "note" | "notesplain" | "comments" => Column::Notes, + "favorite" | "favourite" | "fav" => Column::Favorite, + "tags" | "folder" => Column::Tags, + "archived" => Column::Archived, + _ => Column::Other, + } +} + +/// A parsed CSV whose cells are wiped on drop. +pub struct Records(pub Vec>); + +impl Drop for Records { + fn drop(&mut self) { + for row in &mut self.0 { + row.iter_mut().for_each(Zeroize::zeroize); + } + } +} + +/// RFC 4180 reader. Blank lines are skipped. An unterminated quote is an error. +pub fn read_records(text: &str) -> Result { + let mut rows = Records(vec![]); + let mut row: Vec = vec![]; + let mut field = String::new(); + let mut in_quotes = false; + let mut chars = text.chars().peekable(); + while let Some(c) = chars.next() { + if in_quotes { + match c { + '"' if chars.peek() == Some(&'"') => { + chars.next(); + field.push('"'); + } + '"' => in_quotes = false, + _ => field.push(c), + } + continue; + } + match c { + '"' if field.is_empty() => in_quotes = true, + ',' => row.push(std::mem::take(&mut field)), + '\r' if chars.peek() == Some(&'\n') => {} + '\n' | '\r' => { + row.push(std::mem::take(&mut field)); + push_row(&mut rows.0, std::mem::take(&mut row)); + } + _ => field.push(c), + } + } + if in_quotes { + field.zeroize(); + row.iter_mut().for_each(Zeroize::zeroize); + return Err(malformed("CSV has an unterminated quoted field")); + } + if !field.is_empty() || !row.is_empty() { + row.push(field); + push_row(&mut rows.0, row); + } + Ok(rows) +} + +fn push_row(rows: &mut Vec>, mut row: Vec) { + if row.iter().all(|f| f.trim().is_empty()) { + row.iter_mut().for_each(Zeroize::zeroize); + } else { + rows.push(row); + } +} + +fn truthy(s: &str) -> bool { + matches!(s.trim().to_lowercase().as_str(), "1" | "true" | "yes" | "y" | "x") +} + +pub fn parse(bytes: &[u8], now: i64, limits: &Limits) -> Result { + let bytes = bytes.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(bytes); + let text = std::str::from_utf8(bytes).map_err(|_| malformed("CSV is not UTF-8"))?; + let records = read_records(text)?; + let Some((header, rows)) = records.0.split_first() else { return Err(malformed("CSV is empty")) }; + let columns: Vec = header.iter().map(|h| column_for(h)).collect(); + let has = |c: Column| columns.contains(&c); + if !(has(Column::Title) || has(Column::Url)) + || !(has(Column::Password) || has(Column::Username) || has(Column::Notes)) + { + return Err(ImportError::Unsupported( + "CSV header must name a title or url column and a username, password or notes column".into(), + )); + } + limits.check_items(rows.len())?; + + let mut out = Parsed::default(); + let mut archived = 0u32; + for row in rows { + let cell = |c: Column| columns.iter().position(|x| *x == c).and_then(|i| row.get(i)).map_or("", |s| s.as_str()); + let mut item = new_item(ItemKind::Login, cell(Column::Title), now); + if cell(Column::Title).trim().is_empty() { + if let Some(host) = + url::Url::parse(cell(Column::Url).trim()).ok().and_then(|u| u.host_str().map(String::from)) + { + item.title = host; + } + } + set_opt(&mut item.username, cell(Column::Username)); + set_opt(&mut item.password, cell(Column::Password)); + set_opt(&mut item.totp, cell(Column::Otp).trim()); + set_opt(&mut item.notes, cell(Column::Notes)); + item.urls = cell(Column::Url).lines().map(str::trim).filter(|u| !u.is_empty()).map(String::from).collect(); + item.favorite = truthy(cell(Column::Favorite)); + item.tags = + cell(Column::Tags).split([',', ';']).map(str::trim).filter(|t| !t.is_empty()).map(String::from).collect(); + if truthy(cell(Column::Archived)) { + item.tags.push("Archived".into()); + archived += 1; + } + let mut extras = Extras::default(); + for (i, value) in row.iter().enumerate() { + match columns.get(i) { + Some(Column::Other) => extras.push(&header[i], value), + Some(_) => {} + None => extras.push(&format!("Column {}", i + 1), value), + } + } + extras.append_to(&mut item, "— Imported fields —"); + out.items.push(item); + } + if archived > 0 { + out.warnings.push(format!("{archived} archived item(s) were imported with the tag \"Archived\"")); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rfc4180() { + let r = read_records("a,\"b,c\",\"d \"\"q\"\"\"\r\n\r\n\"multi\nline\",,x\n").unwrap(); + assert_eq!(r.0, vec![vec!["a", "b,c", "d \"q\""], vec!["multi\nline", "", "x"]]); + let r = read_records("a,b").unwrap(); + assert_eq!(r.0, vec![vec!["a", "b"]]); + assert!(read_records("a,\"open").is_err()); + } + + #[test] + fn header_aliases() { + assert_eq!(column_for("Login_URI"), Column::Url); + assert_eq!(column_for("OTPAuth"), Column::Otp); + assert_eq!(column_for(" Title "), Column::Title); + assert_eq!(column_for("Security question"), Column::Other); + } +} diff --git a/crates/kv-import/src/dedup.rs b/crates/kv-import/src/dedup.rs new file mode 100644 index 0000000..d5c5150 --- /dev/null +++ b/crates/kv-import/src/dedup.rs @@ -0,0 +1,110 @@ +//! Duplicate detection: an imported item duplicates an existing one (or an earlier one in the +//! same import) when (title, username, primary URL host) match, case-insensitively. + +use std::collections::HashSet; + +use kv_model::Item; + +pub const IMPORTED_SUFFIX: &str = " (imported)"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DuplicatePolicy { + /// Leave the existing item alone and drop the imported one. + Skip, + /// Import it anyway, with `" (imported)"` appended to the title. + AddSuffix, +} + +pub type Key = (String, String, String); + +/// Host of the first URL, lowercased, without a leading `www.`. Empty when there is no usable URL. +pub fn primary_host(item: &Item) -> String { + let Some(raw) = item.urls.first().map(|u| u.trim()) else { return String::new() }; + let parsed = url::Url::parse(raw).ok().filter(|u| u.host_str().is_some()).or_else(|| { + // Bare "example.com/login" from CSVs. + url::Url::parse(&format!("https://{raw}")).ok() + }); + let host = parsed.and_then(|u| u.host_str().map(str::to_lowercase)).unwrap_or_else(|| raw.to_lowercase()); + host.strip_prefix("www.").map(str::to_string).unwrap_or(host) +} + +pub fn key(item: &Item) -> Key { + (item.title.trim().to_lowercase(), item.username.as_deref().unwrap_or("").trim().to_lowercase(), primary_host(item)) +} + +pub struct Plan { + /// Items to store, in input order. + pub to_add: Vec, + /// Indices into the input of the duplicates (skipped or renamed). + pub duplicates: Vec, + pub skipped: u32, + pub renamed: u32, +} + +/// Decides what to store. `existing` are the keys of the items already in the vault. +pub fn plan(items: Vec, existing: impl IntoIterator, policy: DuplicatePolicy) -> Plan { + let mut seen: HashSet = existing.into_iter().collect(); + let mut out = Plan { to_add: Vec::with_capacity(items.len()), duplicates: vec![], skipped: 0, renamed: 0 }; + for (idx, mut item) in items.into_iter().enumerate() { + let k = key(&item); + if !seen.contains(&k) { + seen.insert(k); + out.to_add.push(item); + continue; + } + out.duplicates.push(idx); + match policy { + DuplicatePolicy::Skip => out.skipped += 1, + DuplicatePolicy::AddSuffix => { + item.title.push_str(IMPORTED_SUFFIX); + // Importing the same file twice still yields only one "(imported)" copy per key. + while !seen.insert(key(&item)) { + item.title.push_str(IMPORTED_SUFFIX); + } + out.renamed += 1; + out.to_add.push(item); + } + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn login(title: &str, user: &str, url: &str) -> Item { + let mut i = Item::new_login(title, 0); + i.username = Some(user.into()); + if !url.is_empty() { + i.urls = vec![url.into()]; + } + i + } + + #[test] + fn host_normalization() { + assert_eq!(primary_host(&login("a", "b", "https://WWW.Example.com/login?x=1")), "example.com"); + assert_eq!(primary_host(&login("a", "b", "example.com/login")), "example.com"); + assert_eq!(primary_host(&login("a", "b", "")), ""); + } + + #[test] + fn skip_and_suffix() { + let existing = [key(&login("GitHub", "me", "https://github.com"))]; + let items = vec![ + login("github", "ME", "https://github.com/login"), + login("GitHub", "other", "https://github.com"), + login("Mail", "me", "https://mail.example"), + login("Mail", "me", "https://mail.example/inbox"), + ]; + let p = plan(items.clone(), existing.clone(), DuplicatePolicy::Skip); + assert_eq!((p.to_add.len(), p.skipped, p.renamed), (2, 2, 0)); + assert_eq!(p.duplicates, vec![0, 3]); + + let p = plan(items, existing, DuplicatePolicy::AddSuffix); + assert_eq!((p.to_add.len(), p.skipped, p.renamed), (4, 0, 2)); + assert_eq!(p.to_add[0].title, "github (imported)"); + assert_eq!(p.to_add[3].title, "Mail (imported)"); + } +} diff --git a/crates/kv-import/src/export.rs b/crates/kv-import/src/export.rs new file mode 100644 index 0000000..4b41795 --- /dev/null +++ b/crates/kv-import/src/export.rs @@ -0,0 +1,182 @@ +//! Passphrase-encrypted export (`.kvex`, docs/09 "Encrypted export"). +//! +//! ```text +//! magic "KVEX" | version u8 = 1 | alg u8 = 1 | m_kib u32le | t u32le | p u32le | salt[16] | nonce[24] | ciphertext +//! key = Argon2id(NFKD(trim(passphrase)), salt, m_kib, t, p, L = 32) +//! ciphertext = XChaCha20-Poly1305(key, nonce, aad = all header bytes, pad(JSON)) +//! JSON = {"format": "kestrove-export", "version": 1, "exported_at": , "items": [Item…]} +//! ``` +//! +//! `alg = 1` names the Argon2id + XChaCha20-Poly1305 suite. Readers refuse KDF parameters below +//! the account KDF minimum (m ≥ 64 MiB, t ≥ 3) and above a sanity cap (so a crafted file cannot +//! make the reader allocate gigabytes). Plaintext export is deliberately not offered. + +use argon2::{Algorithm, Argon2, Params, Version}; +use chacha20poly1305::{ + aead::{Aead, KeyInit, Payload}, + Key, XChaCha20Poly1305, XNonce, +}; +use kv_crypto::{aead, random_bytes}; +use kv_model::Item; +use kv_proto::{MIN_KDF_M_KIB, MIN_KDF_T}; +use serde::{Deserialize, Serialize}; +use unicode_normalization::UnicodeNormalization; +use zeroize::Zeroizing; + +use crate::{malformed, ImportError, Result}; + +pub const MAGIC: &[u8; 4] = b"KVEX"; +pub const VERSION: u8 = 1; +pub const ALG_ARGON2ID_XCHACHA20POLY1305: u8 = 1; +pub const HEADER_LEN: usize = 4 + 1 + 1 + 12 + 16 + 24; +pub const MIN_PASSPHRASE_CHARS: usize = 10; +const FORMAT_NAME: &str = "kestrove-export"; + +/// Upper bounds a reader accepts (1 GiB, 10 passes, 16 lanes). +const MAX_M_KIB: u32 = 1024 * 1024; +const MAX_T: u32 = 10; +const MAX_P: u32 = 16; + +/// Argon2id cost for new exports: the account KDF default (RFC 9106 second option). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct KdfCost { + pub m_kib: u32, + pub t: u32, + pub p: u32, +} + +impl Default for KdfCost { + fn default() -> Self { + Self { m_kib: 64 * 1024, t: 3, p: 4 } + } +} + +impl KdfCost { + fn validate(&self) -> Result<()> { + if self.m_kib < MIN_KDF_M_KIB || self.t < MIN_KDF_T || self.p == 0 { + return Err(ImportError::Unsupported("KDF parameters below the minimum".into())); + } + if self.m_kib > MAX_M_KIB || self.t > MAX_T || self.p > MAX_P { + return Err(ImportError::Unsupported("KDF parameters above the allowed maximum".into())); + } + Ok(()) + } +} + +#[derive(Serialize)] +struct DocOut<'a> { + format: &'a str, + version: u32, + exported_at: i64, + items: &'a [Item], +} + +#[derive(Deserialize)] +struct DocIn { + format: String, + version: u32, + items: Vec, +} + +fn derive_key(passphrase: &str, salt: &[u8; 16], cost: KdfCost) -> Result> { + let normalized = Zeroizing::new(passphrase.trim().nfkd().collect::()); + let params = Params::new(cost.m_kib, cost.t, cost.p, Some(32)).map_err(|_| ImportError::Crypto)?; + let mut key = Zeroizing::new([0u8; 32]); + Argon2::new(Algorithm::Argon2id, Version::V0x13, params) + .hash_password_into(normalized.as_bytes(), salt, key.as_mut()) + .map_err(|_| ImportError::Crypto)?; + Ok(key) +} + +fn header(cost: KdfCost, salt: &[u8; 16], nonce: &[u8; 24]) -> Vec { + let mut h = Vec::with_capacity(HEADER_LEN); + h.extend_from_slice(MAGIC); + h.push(VERSION); + h.push(ALG_ARGON2ID_XCHACHA20POLY1305); + h.extend_from_slice(&cost.m_kib.to_le_bytes()); + h.extend_from_slice(&cost.t.to_le_bytes()); + h.extend_from_slice(&cost.p.to_le_bytes()); + h.extend_from_slice(salt); + h.extend_from_slice(nonce); + h +} + +/// Encrypts `items` under `passphrase` with the default cost. +pub fn encrypt(items: &[Item], passphrase: &str, now: i64) -> Result> { + encrypt_with_cost(items, passphrase, now, KdfCost::default()) +} + +pub fn encrypt_with_cost(items: &[Item], passphrase: &str, now: i64, cost: KdfCost) -> Result> { + if passphrase.trim().chars().count() < MIN_PASSPHRASE_CHARS { + return Err(ImportError::WeakPassphrase { min: MIN_PASSPHRASE_CHARS }); + } + cost.validate()?; + let salt = random_bytes::<16>(); + let nonce = random_bytes::<24>(); + let key = derive_key(passphrase, &salt, cost)?; + let json = Zeroizing::new( + serde_json::to_vec(&DocOut { format: FORMAT_NAME, version: 1, exported_at: now, items }).map_err(malformed)?, + ); + let padded = aead::pad(&json); + let mut out = header(cost, &salt, &nonce); + let ct = XChaCha20Poly1305::new(Key::from_slice(key.as_ref())) + .encrypt(XNonce::from_slice(&nonce), Payload { msg: &padded, aad: &out }) + .map_err(|_| ImportError::Crypto)?; + out.extend_from_slice(&ct); + Ok(out) +} + +/// True if `bytes` starts like an encrypted export. +pub fn looks_like_export(bytes: &[u8]) -> bool { + bytes.starts_with(MAGIC) +} + +pub fn decrypt(bytes: &[u8], passphrase: &str) -> Result> { + if bytes.len() < HEADER_LEN + 16 || !looks_like_export(bytes) { + return Err(malformed("not a Kestrove encrypted export")); + } + let (hdr, ct) = bytes.split_at(HEADER_LEN); + if hdr[4] != VERSION || hdr[5] != ALG_ARGON2ID_XCHACHA20POLY1305 { + return Err(ImportError::Unsupported(format!("export version {} / alg {}", hdr[4], hdr[5]))); + } + let u32_at = |o: usize| u32::from_le_bytes([hdr[o], hdr[o + 1], hdr[o + 2], hdr[o + 3]]); + let cost = KdfCost { m_kib: u32_at(6), t: u32_at(10), p: u32_at(14) }; + cost.validate()?; + let mut salt = [0u8; 16]; + salt.copy_from_slice(&hdr[18..34]); + let key = derive_key(passphrase, &salt, cost)?; + let padded = Zeroizing::new( + XChaCha20Poly1305::new(Key::from_slice(key.as_ref())) + .decrypt(XNonce::from_slice(&hdr[34..58]), Payload { msg: ct, aad: hdr }) + .map_err(|_| ImportError::BadPassphrase)?, + ); + let json = aead::unpad(&padded).map_err(|_| malformed("export padding"))?; + let doc: DocIn = serde_json::from_slice(&json).map_err(|e| malformed(format!("export JSON: {e}")))?; + if doc.format != FORMAT_NAME || doc.version != 1 { + return Err(ImportError::Unsupported(format!("{} version {}", doc.format, doc.version))); + } + Ok(doc.items) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn header_layout() { + assert_eq!(header(KdfCost::default(), &[0; 16], &[0; 24]).len(), HEADER_LEN); + } + + #[test] + fn rejects_weak_and_huge_params() { + assert!(KdfCost { m_kib: 1024, t: 3, p: 1 }.validate().is_err()); + assert!(KdfCost { m_kib: 64 * 1024, t: 1, p: 1 }.validate().is_err()); + assert!(KdfCost { m_kib: 4 * 1024 * 1024, t: 3, p: 1 }.validate().is_err()); + assert!(KdfCost::default().validate().is_ok()); + } + + #[test] + fn short_passphrase() { + assert_eq!(encrypt(&[], "short", 0).unwrap_err(), ImportError::WeakPassphrase { min: 10 }); + } +} diff --git a/crates/kv-import/src/lib.rs b/crates/kv-import/src/lib.rs new file mode 100644 index 0000000..ba5a2c3 --- /dev/null +++ b/crates/kv-import/src/lib.rs @@ -0,0 +1,223 @@ +//! Import from other password managers and passphrase-encrypted export +//! (kestrove-spec/docs/09-import-export.md). +//! +//! This crate is pure: it turns bytes into [`Item`]s (and back, for the encrypted export). +//! Storing, sealing and seq-stamping happen in `kv-ffi`, through the normal save path. +//! +//! Rules: +//! * Untrusted input never panics: every parser returns [`ImportError`]. +//! * Every input is size-capped before parsing ([`Limits`]); ZIP entries are capped on +//! their *decompressed* size and the archive on its entry count. +//! * Plaintext buffers are zeroized ([`zeroize::Zeroizing`], wiped JSON values, wiped CSV cells). +//! * Nothing is silently dropped: fields without a home in [`Item`] go into the notes, and what +//! cannot be imported at all (attachments, password history) is reported in the warnings. + +pub mod csv; +pub mod dedup; +pub mod export; +pub mod onepux; + +use kv_model::{Item, ItemKind}; +use thiserror::Error; +use uuid::Uuid; +use zeroize::{Zeroize, ZeroizeOnDrop}; + +pub use dedup::{plan, DuplicatePolicy, Plan}; + +#[derive(Debug, Error, PartialEq, Eq)] +pub enum ImportError { + #[error("input is larger than the {limit}-byte limit")] + TooLarge { limit: u64 }, + #[error("archive has more than {limit} entries")] + TooManyEntries { limit: usize }, + #[error("malformed input: {0}")] + Malformed(String), + #[error("wrong passphrase or corrupted export")] + BadPassphrase, + #[error("export passphrase must be at least {min} characters")] + WeakPassphrase { min: usize }, + #[error("unsupported export: {0}")] + Unsupported(String), + #[error("crypto failure")] + Crypto, +} + +pub type Result = std::result::Result; + +fn malformed(e: impl std::fmt::Display) -> ImportError { + ImportError::Malformed(e.to_string()) +} + +/// Input caps. [`Limits::default`] is what the apps get; tests use smaller ones. +#[derive(Debug, Clone, Copy)] +pub struct Limits { + /// Raw input (zip, CSV or encrypted export). + pub max_input: u64, + /// Decompressed size of `export.data` inside a `.1pux`. + pub max_decompressed: u64, + /// Entries in a `.1pux` archive (including `files/…`). + pub max_entries: usize, + /// Items produced by one import. + pub max_items: usize, +} + +impl Default for Limits { + fn default() -> Self { + Self { + max_input: 100 * 1024 * 1024, + max_decompressed: 64 * 1024 * 1024, + max_entries: 10_000, + max_items: 100_000, + } + } +} + +impl Limits { + fn check_input(&self, len: usize) -> Result<()> { + if len as u64 > self.max_input { + return Err(ImportError::TooLarge { limit: self.max_input }); + } + Ok(()) + } + + fn check_items(&self, n: usize) -> Result<()> { + if n > self.max_items { + return Err(ImportError::Malformed(format!("more than {} items", self.max_items))); + } + Ok(()) + } +} + +/// What to parse. +#[derive(Clone, Zeroize, ZeroizeOnDrop)] +pub enum Format { + /// 1Password Unencrypted Export (`.1pux`, a ZIP). + OnePux, + /// 1Password CSV or a generic `title,url,username,password,otp,notes` CSV. + Csv, + /// A Kestrove encrypted export ([`export`]). + Encrypted { passphrase: String }, +} + +/// Result of parsing. Items have fresh ids and `seq = 1`; nothing is stored yet. +#[derive(Default)] +pub struct Parsed { + pub items: Vec, + /// Human-readable, may name items or files, never contains a secret value. + pub warnings: Vec, + pub attachments_skipped: u32, +} + +/// Counts only: never prints item contents. +impl std::fmt::Debug for Parsed { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Parsed") + .field("items", &self.items.len()) + .field("warnings", &self.warnings) + .field("attachments_skipped", &self.attachments_skipped) + .finish() + } +} + +impl Parsed { + pub fn count(&self, kind: ItemKind) -> u32 { + self.items.iter().filter(|i| i.kind == kind).count() as u32 + } +} + +pub fn parse(bytes: &[u8], format: &Format, now: i64) -> Result { + parse_with_limits(bytes, format, now, &Limits::default()) +} + +pub fn parse_with_limits(bytes: &[u8], format: &Format, now: i64, limits: &Limits) -> Result { + limits.check_input(bytes.len())?; + let parsed = match format { + Format::OnePux => onepux::parse(bytes, now, limits)?, + Format::Csv => csv::parse(bytes, now, limits)?, + Format::Encrypted { passphrase } => { + let mut items = export::decrypt(bytes, passphrase)?; + limits.check_items(items.len())?; + for i in &mut items { + // A restored item is a new item in this vault: new id, fresh edit counter. + i.id = Uuid::new_v4(); + i.seq = 1; + } + Parsed { items, ..Default::default() } + } + }; + limits.check_items(parsed.items.len())?; + Ok(parsed) +} + +/// A blank item of `kind`. +pub(crate) fn new_item(kind: ItemKind, title: &str, now: i64) -> Item { + let title = title.trim(); + let mut item = Item::new_login(if title.is_empty() { "Untitled" } else { title }, now); + item.kind = kind; + item +} + +/// Collects `label: value` lines that have no field of their own and appends them to the notes. +#[derive(Default)] +pub(crate) struct Extras { + lines: String, +} + +impl Extras { + pub fn heading(&mut self, title: &str) { + let title = title.trim(); + if !title.is_empty() { + if !self.lines.is_empty() { + self.lines.push('\n'); + } + self.lines.push('['); + self.lines.push_str(title); + self.lines.push_str("]\n"); + } + } + + pub fn push(&mut self, label: &str, value: &str) { + if value.trim().is_empty() { + return; + } + let label = label.trim(); + self.lines.push_str(if label.is_empty() { "Field" } else { label }); + self.lines.push_str(": "); + self.lines.push_str(value); + self.lines.push('\n'); + } + + pub fn has_values(&self) -> bool { + self.lines.contains(": ") + } + + /// Appends the collected lines under `banner` (only if any value was pushed). + pub fn append_to(mut self, item: &mut Item, banner: &str) { + if self.has_values() { + let mut notes = item.notes.take().unwrap_or_default(); + let trimmed_len = notes.trim_end().len(); + notes.truncate(trimmed_len); + if !notes.is_empty() { + notes.push_str("\n\n"); + } + notes.push_str(banner); + notes.push('\n'); + notes.push_str(self.lines.trim_end()); + item.notes = Some(notes); + } + self.lines.zeroize(); + } +} + +impl Drop for Extras { + fn drop(&mut self) { + self.lines.zeroize(); + } +} + +/// Puts `s` into `slot` unless it is blank. Kept verbatim: a password may start or end with a space. +pub(crate) fn set_opt(slot: &mut Option, s: &str) { + if !s.trim().is_empty() { + *slot = Some(s.to_string()); + } +} diff --git a/crates/kv-import/src/onepux.rs b/crates/kv-import/src/onepux.rs new file mode 100644 index 0000000..ef1ef57 --- /dev/null +++ b/crates/kv-import/src/onepux.rs @@ -0,0 +1,297 @@ +//! 1Password Unencrypted Export (`.1pux`): a ZIP holding `export.attributes`, `export.data` +//! (JSON: `accounts[].vaults[].items[]`) and `files/` (attachments, not imported). +//! +//! Mapping (docs/09): Login 001 and Password 005 → login; Credit Card 002 → card; Secure Note 003 +//! → secure note; Identity 004 → identity; every other category → secure note. Fields without a +//! home in `Item` (card and identity fields, custom sections, extra form fields) are appended to +//! the notes as `Label: value` lines under a "— Imported from 1Password —" banner. + +use std::io::{Cursor, Read}; + +use kv_model::{Item, ItemKind}; +use serde_json::Value; +use zeroize::{Zeroize, Zeroizing}; + +use crate::{malformed, new_item, set_opt, Extras, ImportError, Limits, Parsed, Result}; + +const DATA_ENTRY: &str = "export.data"; +const BANNER: &str = "— Imported from 1Password —"; + +/// JSON whose strings are wiped on drop. +struct Wiped(Value); + +impl Drop for Wiped { + fn drop(&mut self) { + wipe(&mut self.0); + } +} + +fn wipe(v: &mut Value) { + match v { + Value::String(s) => s.zeroize(), + Value::Array(a) => a.iter_mut().for_each(wipe), + Value::Object(m) => m.values_mut().for_each(wipe), + _ => {} + } +} + +/// Reads `export.data` with the entry-count and decompressed-size caps. +pub fn read_export_data(bytes: &[u8], limits: &Limits) -> Result<(Zeroizing>, usize)> { + let mut zip = + zip::ZipArchive::new(Cursor::new(bytes)).map_err(|e| malformed(format!("not a .1pux archive: {e}")))?; + if zip.len() > limits.max_entries { + return Err(ImportError::TooManyEntries { limit: limits.max_entries }); + } + let files = + zip.file_names().filter_map(|n| n.ok()).filter(|n| n.starts_with("files/") && !n.ends_with('/')).count(); + let entry = zip.by_name(DATA_ENTRY).map_err(|_| malformed("export.data is missing"))?; + if entry.size() > limits.max_decompressed { + return Err(ImportError::TooLarge { limit: limits.max_decompressed }); + } + // Never trust the declared size: stop reading one byte past the cap. + let mut out = Zeroizing::new(Vec::new()); + entry + .take(limits.max_decompressed + 1) + .read_to_end(&mut out) + .map_err(|e| malformed(format!("export.data: {e}")))?; + if out.len() as u64 > limits.max_decompressed { + return Err(ImportError::TooLarge { limit: limits.max_decompressed }); + } + Ok((out, files)) +} + +pub fn parse(bytes: &[u8], now: i64, limits: &Limits) -> Result { + let (data, files_in_archive) = read_export_data(bytes, limits)?; + let root = Wiped(serde_json::from_slice(&data).map_err(|e| malformed(format!("export.data: {e}")))?); + let accounts = + root.0.get("accounts").and_then(Value::as_array).ok_or_else(|| malformed("export.data has no accounts"))?; + + let vault_count: usize = + accounts.iter().map(|a| a.get("vaults").and_then(Value::as_array).map_or(0, Vec::len)).sum(); + let mut ctx = Ctx { out: Parsed::default(), now, password_history: 0, other_category: 0, trashed: 0, archived: 0 }; + for account in accounts { + for vault in account.get("vaults").and_then(Value::as_array).into_iter().flatten() { + let vault_name = str_at(vault, &["attrs", "name"]); + for raw in vault.get("items").and_then(Value::as_array).into_iter().flatten() { + limits.check_items(ctx.out.items.len() + 1)?; + if let Some(mut item) = ctx.item(raw)? { + if vault_count > 1 && !vault_name.is_empty() { + item.tags.push(vault_name.to_string()); + } + ctx.out.items.push(item); + } + } + } + } + let Ctx { mut out, password_history, other_category, trashed, archived, .. } = ctx; + if out.attachments_skipped == 0 && files_in_archive > 0 { + out.attachments_skipped = files_in_archive as u32; + out.warnings.push(format!( + "{files_in_archive} file(s) in the archive were not imported (attachments are not supported yet)" + )); + } + if password_history > 0 { + out.warnings.push(format!("{password_history} previous password(s) from password history were not imported")); + } + if other_category > 0 { + out.warnings.push(format!("{other_category} item(s) of other categories were imported as secure notes")); + } + if archived > 0 { + out.warnings.push(format!("{archived} archived item(s) were imported with the tag \"Archived\"")); + } + if trashed > 0 { + out.warnings.push(format!("{trashed} deleted item(s) were skipped")); + } + Ok(out) +} + +struct Ctx { + out: Parsed, + now: i64, + password_history: u32, + other_category: u32, + trashed: u32, + archived: u32, +} + +impl Ctx { + fn item(&mut self, raw: &Value) -> Result> { + if !raw.is_object() { + return Err(malformed("item is not an object")); + } + let state = str_at(raw, &["state"]); + if state == "deleted" || state == "trashed" { + self.trashed += 1; + return Ok(None); + } + let category = str_at(raw, &["categoryUuid"]); + let kind = match category { + "001" | "005" => ItemKind::Login, + "002" => ItemKind::Card, + "003" => ItemKind::SecureNote, + "004" => ItemKind::Identity, + _ => { + self.other_category += 1; + ItemKind::SecureNote + } + }; + let overview = raw.get("overview").unwrap_or(&Value::Null); + let details = raw.get("details").unwrap_or(&Value::Null); + let mut item = new_item(kind, str_at(overview, &["title"]), self.now); + let title = item.title.clone(); + if let Some(t) = raw.get("createdAt").and_then(Value::as_i64) { + item.created_at = t; + } + if let Some(t) = raw.get("updatedAt").and_then(Value::as_i64) { + item.modified_at = t.max(item.created_at); + } + item.favorite = raw.get("favIndex").and_then(Value::as_i64).is_some_and(|f| f > 0); + item.tags = overview + .get("tags") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(String::from) + .collect(); + if state == "archived" { + item.tags.push("Archived".into()); + self.archived += 1; + } + let mut urls: Vec = overview + .get("urls") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|u| u.get("url").and_then(Value::as_str)) + .filter(|u| !u.trim().is_empty()) + .map(String::from) + .collect(); + let main_url = str_at(overview, &["url"]); + if !main_url.is_empty() && !urls.iter().any(|u| u == main_url) { + urls.insert(0, main_url.to_string()); + } + item.urls = urls; + set_opt(&mut item.notes, str_at(details, &["notesPlain"])); + + let mut extras = Extras::default(); + if category == "005" { + set_opt(&mut item.password, str_at(details, &["password"])); + } else if let Some(pw) = details.get("password").and_then(Value::as_str) { + extras.push("Password", pw); + } + for f in details.get("loginFields").and_then(Value::as_array).into_iter().flatten() { + let value = str_at(f, &["value"]); + match str_at(f, &["designation"]) { + "username" if item.username.is_none() && kind == ItemKind::Login => set_opt(&mut item.username, value), + "password" if item.password.is_none() && kind == ItemKind::Login => set_opt(&mut item.password, value), + _ => { + let name = str_at(f, &["name"]); + extras.push(if name.is_empty() { "Form field" } else { name }, value); + } + } + } + for section in details.get("sections").and_then(Value::as_array).into_iter().flatten() { + let section_title = str_at(section, &["title"]); + extras.heading(section_title); + for field in section.get("fields").and_then(Value::as_array).into_iter().flatten() { + self.field(&mut item, &mut extras, &title, field); + } + } + if let Some(doc) = details.get("documentAttributes").filter(|d| d.is_object()) { + self.attachment(&title, doc); + } + self.password_history += details.get("passwordHistory").and_then(Value::as_array).map_or(0, |h| h.len() as u32); + extras.append_to(&mut item, BANNER); + Ok(Some(item)) + } + + fn field(&mut self, item: &mut Item, extras: &mut Extras, title: &str, field: &Value) { + let label = { + let t = str_at(field, &["title"]); + if t.is_empty() { + str_at(field, &["id"]) + } else { + t + } + }; + let Some(value) = field.get("value").and_then(Value::as_object) else { return }; + for (vtype, v) in value { + match vtype.as_str() { + "totp" if item.totp.is_none() && v.as_str().is_some_and(|s| !s.trim().is_empty()) => { + item.totp = v.as_str().map(|s| s.trim().to_string()); + } + "file" => self.attachment(title, v), + _ => { + let text = Zeroizing::new(render(vtype, v)); + extras.push(label, &text); + } + } + } + } + + fn attachment(&mut self, title: &str, attrs: &Value) { + self.out.attachments_skipped += 1; + let name = str_at(attrs, &["fileName"]); + self.out.warnings.push(format!( + "\"{title}\": attachment \"{}\" was not imported (attachments are not supported yet)", + if name.is_empty() { "unnamed" } else { name } + )); + } +} + +/// Text for a 1Password field value (`{"": }`). +fn render(vtype: &str, v: &Value) -> String { + match (vtype, v) { + (_, Value::String(s)) => s.clone(), + ("date", Value::Number(n)) => n.as_i64().map(format_date).unwrap_or_else(|| n.to_string()), + ("monthYear", Value::Number(n)) => match n.as_i64() { + Some(ym) if ym > 0 => format!("{:02}/{}", ym % 100, ym / 100), + _ => n.to_string(), + }, + ("email", Value::Object(o)) => o.get("email_address").and_then(Value::as_str).unwrap_or("").to_string(), + ("address", Value::Object(o)) => ["street", "city", "state", "zip", "country"] + .iter() + .filter_map(|k| o.get(*k).and_then(Value::as_str)) + .filter(|s| !s.trim().is_empty()) + .collect::>() + .join(", "), + ("sshKey", Value::Object(o)) => o.get("privateKey").and_then(Value::as_str).unwrap_or("").to_string(), + (_, Value::Null) => String::new(), + (_, Value::Bool(b)) => b.to_string(), + (_, Value::Number(n)) => n.to_string(), + // Unknown structured value: keep it verbatim rather than drop it. + (_, other) => other.to_string(), + } +} + +/// `YYYY-MM-DD` (UTC) for a Unix timestamp in seconds. +fn format_date(secs: i64) -> String { + let z = secs.div_euclid(86_400) + 719_468; + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097); + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = yoe + era * 400 + i64::from(m <= 2); + format!("{y:04}-{m:02}-{d:02}") +} + +fn str_at<'a>(v: &'a Value, path: &[&str]) -> &'a str { + path.iter().try_fold(v, |v, k| v.get(k)).and_then(Value::as_str).unwrap_or("") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dates() { + assert_eq!(format_date(0), "1970-01-01"); + assert_eq!(format_date(951_782_400), "2000-02-29"); + assert_eq!(format_date(-86_400), "1969-12-31"); + assert_eq!(render("monthYear", &serde_json::json!(202_712)), "12/2027"); + } +} diff --git a/crates/kv-import/tests/fixtures/1password8.csv b/crates/kv-import/tests/fixtures/1password8.csv new file mode 100644 index 0000000..663b717 --- /dev/null +++ b/crates/kv-import/tests/fixtures/1password8.csv @@ -0,0 +1,5 @@ +Title,Url,Username,Password,OTPAuth,Favorite,Archived,Tags,Notes +Example Git,https://git.example.com/login,alex@example.com,correct-horse-battery-staple,otpauth://totp/Example:alex@example.com?secret=JBSWY3DPEHPK3PXP&issuer=Example,true,false,"dev,work","Main code hosting account. +Second line, with a comma and ""quotes""." +Old Forum,https://forum.example.org,alex," leading-space-pw",,false,true,, +Router,,,router-synthetic-pw,,false,false,,Router admin diff --git a/crates/kv-import/tests/fixtures/1pux-export.data.json b/crates/kv-import/tests/fixtures/1pux-export.data.json new file mode 100644 index 0000000..af368e2 --- /dev/null +++ b/crates/kv-import/tests/fixtures/1pux-export.data.json @@ -0,0 +1,253 @@ +{ + "accounts": [ + { + "attrs": { + "accountName": "Synthetic Test", + "name": "Alex Example", + "avatar": "", + "email": "alex@example.com", + "uuid": "AAAAAAAAAAAAAAAAAAAAAAAAAA", + "domain": "https://example.1password.com/" + }, + "vaults": [ + { + "attrs": { "uuid": "vaultpersonal00000000000000", "desc": "", "avatar": "", "name": "Personal", "type": "P" }, + "items": [ + { + "uuid": "item0login0totp000000000000", + "favIndex": 1, + "createdAt": 1614298956, + "updatedAt": 1700000000, + "state": "active", + "categoryUuid": "001", + "details": { + "loginFields": [ + { "value": "alex@example.com", "id": "", "name": "email", "fieldType": "E", "designation": "username" }, + { "value": "correct-horse-battery-staple", "id": "", "name": "password", "fieldType": "P", "designation": "password" }, + { "value": "", "id": "", "name": "remember", "fieldType": "C", "designation": "" } + ], + "notesPlain": "Main code hosting account.", + "sections": [ + { + "title": "", + "name": "add more", + "fields": [ + { + "title": "one-time password", + "id": "TOTP_0001", + "value": { "totp": "otpauth://totp/Example:alex@example.com?secret=JBSWY3DPEHPK3PXP&issuer=Example" }, + "indexAtSource": 0, "guarded": false, "multiline": false, "dontGenerate": false, + "inputTraits": { "keyboard": "default", "correction": "default", "capitalization": "default" } + } + ] + }, + { + "title": "Security questions", + "name": "linked items", + "fields": [ + { "title": "First pet", "id": "q1", "value": { "concealed": "Synthetic Fluffy" }, "indexAtSource": 0, "guarded": false, "multiline": false, "dontGenerate": true, "inputTraits": {} }, + { "title": "Recovery codes", "id": "rc", "value": { "string": "1111-2222\n3333-4444" }, "indexAtSource": 1, "guarded": false, "multiline": true, "dontGenerate": false, "inputTraits": {} } + ] + } + ], + "passwordHistory": [ { "value": "old-synthetic-password", "time": 1600000000 } ] + }, + "overview": { + "subtitle": "alex@example.com", + "urls": [ + { "label": "", "url": "https://git.example.com/login" }, + { "label": "website", "url": "https://www.example.com" } + ], + "title": "Example Git", + "url": "https://git.example.com/login", + "ps": 100, "pbe": 86.2, "pgrng": true, + "tags": ["dev", "work"] + } + }, + { + "uuid": "item1login0bare000000000000", + "favIndex": 0, + "createdAt": 1620000000, + "updatedAt": 1620000000, + "state": "archived", + "categoryUuid": "001", + "details": { + "loginFields": [ + { "value": "alex", "id": "", "name": "user", "fieldType": "T", "designation": "username" }, + { "value": " leading-space-pw", "id": "", "name": "pass", "fieldType": "P", "designation": "password" } + ], + "notesPlain": "", + "sections": [ + { "title": "", "name": "", "fields": [ { "title": "2FA", "id": "otp2", "value": { "totp": "GEZDGNBVGY3TQOJQ" }, "indexAtSource": 0, "guarded": false, "multiline": false, "dontGenerate": false, "inputTraits": {} } ] } + ], + "passwordHistory": [] + }, + "overview": { "subtitle": "alex", "urls": [ { "label": "", "url": "https://forum.example.org" } ], "title": "Old Forum", "url": "https://forum.example.org", "ps": 40, "pbe": 30.0, "pgrng": false } + }, + { + "uuid": "item2card000000000000000000", + "favIndex": 0, + "createdAt": 1630000000, + "updatedAt": 1630000000, + "state": "active", + "categoryUuid": "002", + "details": { + "loginFields": [], + "notesPlain": "Travel card.", + "sections": [ + { + "title": "", + "name": "", + "fields": [ + { "title": "cardholder name", "id": "cardholder", "value": { "string": "ALEX EXAMPLE" }, "indexAtSource": 0, "guarded": true, "multiline": false, "dontGenerate": false, "inputTraits": {} }, + { "title": "type", "id": "type", "value": { "creditCardType": "visa" }, "indexAtSource": 1, "guarded": true, "multiline": false, "dontGenerate": false, "inputTraits": {} }, + { "title": "number", "id": "ccnum", "value": { "creditCardNumber": "4111111111111111" }, "indexAtSource": 2, "guarded": true, "multiline": false, "dontGenerate": false, "inputTraits": {} }, + { "title": "verification number", "id": "cvv", "value": { "concealed": "123" }, "indexAtSource": 3, "guarded": true, "multiline": false, "dontGenerate": true, "inputTraits": {} }, + { "title": "expiry date", "id": "expiry", "value": { "monthYear": 202712 }, "indexAtSource": 4, "guarded": true, "multiline": false, "dontGenerate": false, "inputTraits": {} } + ] + }, + { + "title": "Contact Information", + "name": "contactInfo", + "fields": [ + { "title": "issuing bank", "id": "bank", "value": { "string": "Example Bank" }, "indexAtSource": 0, "guarded": false, "multiline": false, "dontGenerate": false, "inputTraits": {} }, + { "title": "phone (intl)", "id": "phoneIntl", "value": { "phone": "+1 555 0100" }, "indexAtSource": 1, "guarded": false, "multiline": false, "dontGenerate": false, "inputTraits": {} } + ] + } + ], + "passwordHistory": [] + }, + "overview": { "subtitle": "4111 **** 1111", "title": "Example Visa", "url": "", "ps": 0, "pbe": 0.0, "pgrng": false } + }, + { + "uuid": "item3note000000000000000000", + "favIndex": 0, + "createdAt": 1640000000, + "updatedAt": 1640000000, + "state": "active", + "categoryUuid": "003", + "details": { "loginFields": [], "notesPlain": "Wi-Fi: SyntheticNet\nKey: not-a-real-key", "sections": [], "passwordHistory": [] }, + "overview": { "subtitle": "", "title": "Home Wi-Fi", "url": "", "ps": 0, "pbe": 0.0, "pgrng": false, "tags": ["home"] } + }, + { + "uuid": "item4identity00000000000000", + "favIndex": 0, + "createdAt": 1650000000, + "updatedAt": 1650000000, + "state": "active", + "categoryUuid": "004", + "details": { + "loginFields": [], + "notesPlain": "", + "sections": [ + { + "title": "Identification", + "name": "name", + "fields": [ + { "title": "first name", "id": "firstname", "value": { "string": "Alex" }, "indexAtSource": 0, "guarded": false, "multiline": false, "dontGenerate": false, "inputTraits": {} }, + { "title": "last name", "id": "lastname", "value": { "string": "Example" }, "indexAtSource": 1, "guarded": false, "multiline": false, "dontGenerate": false, "inputTraits": {} }, + { "title": "birth date", "id": "birthdate", "value": { "date": 951782400 }, "indexAtSource": 2, "guarded": false, "multiline": false, "dontGenerate": false, "inputTraits": {} } + ] + }, + { + "title": "Address", + "name": "address", + "fields": [ + { "title": "address", "id": "address", "value": { "address": { "street": "1 Example Way", "city": "Sampletown", "country": "us", "zip": "00000", "state": "CA" } }, "indexAtSource": 0, "guarded": false, "multiline": false, "dontGenerate": false, "inputTraits": {} }, + { "title": "phone", "id": "defphone", "value": { "phone": "+1 555 0101" }, "indexAtSource": 1, "guarded": false, "multiline": false, "dontGenerate": false, "inputTraits": {} } + ] + }, + { + "title": "Internet Details", + "name": "internet", + "fields": [ + { "title": "email", "id": "email", "value": { "email": { "email_address": "alex@example.com", "provider": null } }, "indexAtSource": 0, "guarded": false, "multiline": false, "dontGenerate": false, "inputTraits": {} } + ] + } + ], + "passwordHistory": [] + }, + "overview": { "subtitle": "Alex Example", "title": "Alex Example", "url": "", "ps": 0, "pbe": 0.0, "pgrng": false } + }, + { + "uuid": "item5attachment000000000000", + "favIndex": 0, + "createdAt": 1660000000, + "updatedAt": 1660000000, + "state": "active", + "categoryUuid": "001", + "details": { + "loginFields": [ + { "value": "alex", "id": "", "name": "username", "fieldType": "T", "designation": "username" }, + { "value": "insurance-synthetic-pw", "id": "", "name": "password", "fieldType": "P", "designation": "password" } + ], + "notesPlain": "Policy scan attached.", + "sections": [ + { + "title": "", + "name": "", + "fields": [ + { "title": "policy.pdf", "id": "file1", "value": { "file": { "fileName": "policy.pdf", "documentId": "doc0000000000000000000000a", "decryptedSize": 1234 } }, "indexAtSource": 0, "guarded": false, "multiline": false, "dontGenerate": false, "inputTraits": {} } + ] + } + ], + "passwordHistory": [] + }, + "overview": { "subtitle": "alex", "urls": [ { "label": "", "url": "https://insurance.example.net" } ], "title": "Example Insurance", "url": "https://insurance.example.net", "ps": 60, "pbe": 50.0, "pgrng": false } + }, + { + "uuid": "item6password000000000000000", + "favIndex": 0, + "createdAt": 1670000000, + "updatedAt": 1670000000, + "state": "active", + "categoryUuid": "005", + "details": { "loginFields": [], "notesPlain": "Router admin", "sections": [], "passwordHistory": [], "password": "router-synthetic-pw" }, + "overview": { "subtitle": "", "title": "Router", "url": "", "ps": 50, "pbe": 40.0, "pgrng": true } + }, + { + "uuid": "item7trashed000000000000000", + "favIndex": 0, + "createdAt": 1670000000, + "updatedAt": 1670000000, + "state": "deleted", + "categoryUuid": "001", + "details": { "loginFields": [], "notesPlain": "", "sections": [], "passwordHistory": [] }, + "overview": { "subtitle": "", "title": "Trashed", "url": "", "ps": 0, "pbe": 0.0, "pgrng": false } + } + ] + }, + { + "attrs": { "uuid": "vaultshared0000000000000000", "desc": "", "avatar": "", "name": "Shared", "type": "E" }, + "items": [ + { + "uuid": "item8server0000000000000000", + "favIndex": 0, + "createdAt": 1680000000, + "updatedAt": 1680000000, + "state": "active", + "categoryUuid": "110", + "details": { + "loginFields": [], + "notesPlain": "", + "sections": [ + { + "title": "", + "name": "", + "fields": [ + { "title": "URL", "id": "url", "value": { "string": "ssh://build.example.com" }, "indexAtSource": 0, "guarded": false, "multiline": false, "dontGenerate": false, "inputTraits": {} }, + { "title": "username", "id": "username", "value": { "string": "deploy" }, "indexAtSource": 1, "guarded": false, "multiline": false, "dontGenerate": false, "inputTraits": {} }, + { "title": "password", "id": "password", "value": { "concealed": "server-synthetic-pw" }, "indexAtSource": 2, "guarded": false, "multiline": false, "dontGenerate": false, "inputTraits": {} } + ] + } + ], + "passwordHistory": [] + }, + "overview": { "subtitle": "", "title": "Build Server", "url": "", "ps": 0, "pbe": 0.0, "pgrng": false } + } + ] + } + ] + } + ] +} diff --git a/crates/kv-import/tests/fixtures/generic-bom-crlf.csv b/crates/kv-import/tests/fixtures/generic-bom-crlf.csv new file mode 100644 index 0000000..27ee208 --- /dev/null +++ b/crates/kv-import/tests/fixtures/generic-bom-crlf.csv @@ -0,0 +1,4 @@ +NAME,Website,Login,PASSWORD,TOTP,Note,Security Question +Mail,mail.example.com,alex,mail-synthetic-pw,JBSWY3DPEHPK3PXP,"multi +line",Synthetic Fluffy +,https://bank.example.com/login,alex,bank-synthetic-pw,,, diff --git a/crates/kv-import/tests/import.rs b/crates/kv-import/tests/import.rs new file mode 100644 index 0000000..5041891 --- /dev/null +++ b/crates/kv-import/tests/import.rs @@ -0,0 +1,316 @@ +//! Fixture-based tests. All fixture data is synthetic (example.com domains, test card numbers). + +use std::io::{Cursor, Write}; + +use kv_import::{export, parse, parse_with_limits, Format, ImportError, Limits, Parsed}; +use kv_model::{Item, ItemKind}; +use zip::write::SimpleFileOptions; + +const NOW: i64 = 1_760_000_000; +const EXPORT_DATA: &str = include_str!("fixtures/1pux-export.data.json"); + +/// Builds a .1pux the way 1Password lays it out. +fn onepux(entries: &[(&str, &[u8])]) -> Vec { + let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); + let opts = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); + for (name, data) in entries { + if name.ends_with('/') { + zip.add_directory(*name, opts).unwrap(); + } else { + zip.start_file(*name, opts).unwrap(); + zip.write_all(data).unwrap(); + } + } + zip.finish().unwrap().into_inner() +} + +fn fixture_1pux() -> Vec { + onepux(&[ + ("export.attributes", br#"{"version":3,"description":"1Password Unencrypted Export","createdAt":1700000000}"#), + ("export.data", EXPORT_DATA.as_bytes()), + ("files/", b""), + ("files/doc0000000000000000000000a__policy.pdf", b"%PDF-1.4 synthetic"), + ]) +} + +fn by_title<'a>(p: &'a Parsed, title: &str) -> &'a Item { + p.items.iter().find(|i| i.title == title).unwrap_or_else(|| panic!("no item {title}")) +} + +#[test] +fn onepux_fixture() { + let p = parse(&fixture_1pux(), &Format::OnePux, NOW).unwrap(); + // 9 items, one deleted. + assert_eq!(p.items.len(), 8); + assert_eq!(p.count(ItemKind::Login), 4); + assert_eq!(p.count(ItemKind::Card), 1); + assert_eq!(p.count(ItemKind::Identity), 1); + assert_eq!(p.count(ItemKind::SecureNote), 2); + + let git = by_title(&p, "Example Git"); + assert_eq!(git.kind, ItemKind::Login); + assert_eq!(git.username.as_deref(), Some("alex@example.com")); + assert_eq!(git.password.as_deref(), Some("correct-horse-battery-staple")); + assert_eq!(git.urls, vec!["https://git.example.com/login", "https://www.example.com"]); + assert!(git.totp.as_deref().unwrap().starts_with("otpauth://totp/Example:")); + assert!(git.favorite); + assert_eq!(git.tags, vec!["dev", "work", "Personal"]); + assert_eq!(git.created_at, 1_614_298_956); + assert_eq!(git.modified_at, 1_700_000_000); + assert_eq!(git.seq, 1); + let notes = git.notes.as_deref().unwrap(); + assert!(notes.starts_with("Main code hosting account.\n\n— Imported from 1Password —")); + assert!(notes.contains("[Security questions]\nFirst pet: Synthetic Fluffy\nRecovery codes: 1111-2222\n3333-4444")); + + let forum = by_title(&p, "Old Forum"); + assert_eq!(forum.password.as_deref(), Some(" leading-space-pw"), "passwords are kept verbatim"); + assert_eq!(forum.totp.as_deref(), Some("GEZDGNBVGY3TQOJQ")); + assert!(forum.tags.contains(&"Archived".to_string())); + assert!(forum.notes.is_none()); + + let card = by_title(&p, "Example Visa"); + assert_eq!(card.kind, ItemKind::Card); + let n = card.notes.as_deref().unwrap(); + for line in [ + "Travel card.", + "cardholder name: ALEX EXAMPLE", + "type: visa", + "number: 4111111111111111", + "verification number: 123", + "expiry date: 12/2027", + "[Contact Information]\nissuing bank: Example Bank", + ] { + assert!(n.contains(line), "card notes missing {line:?}:\n{n}"); + } + + let note = by_title(&p, "Home Wi-Fi"); + assert_eq!(note.kind, ItemKind::SecureNote); + assert_eq!(note.notes.as_deref(), Some("Wi-Fi: SyntheticNet\nKey: not-a-real-key")); + + let id = by_title(&p, "Alex Example"); + assert_eq!(id.kind, ItemKind::Identity); + let n = id.notes.as_deref().unwrap(); + assert!(n.contains("first name: Alex")); + assert!(n.contains("birth date: 2000-02-29")); + assert!(n.contains("address: 1 Example Way, Sampletown, CA, 00000, us")); + assert!(n.contains("email: alex@example.com")); + + let router = by_title(&p, "Router"); + assert_eq!(router.kind, ItemKind::Login); + assert_eq!(router.password.as_deref(), Some("router-synthetic-pw")); + + let server = by_title(&p, "Build Server"); + assert_eq!(server.kind, ItemKind::SecureNote); + assert!(server.notes.as_deref().unwrap().contains("password: server-synthetic-pw")); + assert_eq!(server.tags, vec!["Shared"]); + + // The attachment is reported, not imported. + assert_eq!(p.attachments_skipped, 1); + assert!(p.warnings.iter().any(|w| w.contains("\"Example Insurance\": attachment \"policy.pdf\" was not imported"))); + assert!(p.warnings.iter().any(|w| w.contains("1 previous password"))); + assert!(p.warnings.iter().any(|w| w.contains("1 deleted item"))); + assert!(p.warnings.iter().any(|w| w.contains("1 item(s) of other categories"))); + // Warnings never carry secret values. + for w in &p.warnings { + for secret in ["correct-horse", "4111", "server-synthetic-pw", "old-synthetic-password"] { + assert!(!w.contains(secret), "warning leaks a secret: {w}"); + } + } +} + +#[test] +fn onepux_unreferenced_files_are_reported() { + let data = br#"{"accounts":[{"attrs":{},"vaults":[{"attrs":{"name":"Personal"},"items":[]}]}]}"#; + let zip = onepux(&[("export.data", data), ("files/x__a.bin", b"1"), ("files/y__b.bin", b"2")]); + let p = parse(&zip, &Format::OnePux, NOW).unwrap(); + assert!(p.items.is_empty()); + assert_eq!(p.attachments_skipped, 2); +} + +#[test] +fn zip_bomb_decompressed_size() { + // 8 MiB of zeros compresses to a few KiB. + let zeros = vec![b' '; 8 * 1024 * 1024]; + let zip = onepux(&[("export.data", &zeros)]); + assert!(zip.len() < 100 * 1024); + let limits = Limits { max_decompressed: 1024 * 1024, ..Limits::default() }; + assert_eq!( + parse_with_limits(&zip, &Format::OnePux, NOW, &limits).unwrap_err(), + ImportError::TooLarge { limit: 1024 * 1024 } + ); +} + +#[test] +fn zip_bomb_entry_count() { + let names: Vec = (0..50).map(|i| format!("files/{i}__f")).collect(); + let mut entries: Vec<(&str, &[u8])> = names.iter().map(|n| (n.as_str(), &b""[..])).collect(); + entries.push(("export.data", b"{\"accounts\":[]}")); + let limits = Limits { max_entries: 20, ..Limits::default() }; + assert_eq!( + parse_with_limits(&onepux(&entries), &Format::OnePux, NOW, &limits).unwrap_err(), + ImportError::TooManyEntries { limit: 20 } + ); +} + +#[test] +fn input_size_cap() { + let limits = Limits { max_input: 10, ..Limits::default() }; + assert_eq!( + parse_with_limits(&[0u8; 11], &Format::Csv, NOW, &limits).unwrap_err(), + ImportError::TooLarge { limit: 10 } + ); +} + +#[test] +fn item_count_cap() { + let csv = "title,password\na,1\nb,2\nc,3\n"; + let limits = Limits { max_items: 2, ..Limits::default() }; + assert!(matches!(parse_with_limits(csv.as_bytes(), &Format::Csv, NOW, &limits), Err(ImportError::Malformed(_)))); +} + +#[test] +fn malformed_inputs_error_without_panicking() { + let full = fixture_1pux(); + let cases: Vec<(Vec, &str)> = vec![ + (vec![], "empty"), + (b"not a zip at all".to_vec(), "garbage"), + (full[..full.len() / 2].to_vec(), "truncated zip"), + (onepux(&[("other.json", b"{}")]), "no export.data"), + (onepux(&[("export.data", b"{not json")]), "bad json"), + (onepux(&[("export.data", b"[1,2,3]")]), "no accounts"), + (onepux(&[("export.data", br#"{"accounts":[{"vaults":[{"items":[42]}]}]}"#)]), "item not object"), + ( + onepux(&[( + "export.data", + br#"{"accounts":[{"vaults":[{"items":[{"categoryUuid":7,"overview":[],"details":{"sections":"x","loginFields":[1,{"value":5}]}}]}]}]}"#, + )]), + "wrong types", + ), + ]; + for (bytes, what) in cases { + let r = std::panic::catch_unwind(|| parse(&bytes, &Format::OnePux, NOW)); + let r = r.unwrap_or_else(|_| panic!("{what}: panicked")); + if what == "wrong types" { + // Tolerated: odd field types are ignored, the item is still imported. + assert_eq!(r.unwrap().items.len(), 1, "{what}"); + } else { + assert!(r.is_err(), "{what}: expected an error"); + } + } + + for (bytes, what) in [ + (&b""[..], "empty csv"), + (b"\xff\xfe", "not utf-8"), + (b"foo,bar\n1,2\n", "unknown header"), + (b"title,password\n\"open,1\n", "unterminated quote"), + ] { + let r = std::panic::catch_unwind(|| parse(bytes, &Format::Csv, NOW)).unwrap_or_else(|_| panic!("{what}")); + assert!(r.is_err(), "{what}"); + } + + for (bytes, what) in [(&b"KVEX"[..], "short export"), (&[0u8; 200][..], "no magic")] { + let r = parse(bytes, &Format::Encrypted { passphrase: "a long passphrase".into() }, NOW); + assert!(matches!(r, Err(ImportError::Malformed(_))), "{what}"); + } +} + +#[test] +fn csv_1password() { + let p = parse(include_bytes!("fixtures/1password8.csv"), &Format::Csv, NOW).unwrap(); + assert_eq!(p.items.len(), 3); + let git = by_title(&p, "Example Git"); + assert_eq!(git.username.as_deref(), Some("alex@example.com")); + assert_eq!(git.password.as_deref(), Some("correct-horse-battery-staple")); + assert_eq!(git.urls, vec!["https://git.example.com/login"]); + assert!(git.totp.as_deref().unwrap().contains("secret=JBSWY3DPEHPK3PXP")); + assert!(git.favorite); + assert_eq!(git.tags, vec!["dev", "work"]); + assert_eq!(git.notes.as_deref(), Some("Main code hosting account.\nSecond line, with a comma and \"quotes\".")); + + let forum = by_title(&p, "Old Forum"); + assert_eq!(forum.password.as_deref(), Some(" leading-space-pw")); + assert_eq!(forum.tags, vec!["Archived"]); + assert!(p.warnings.iter().any(|w| w.contains("1 archived"))); + + let router = by_title(&p, "Router"); + assert!(router.urls.is_empty()); + assert_eq!(router.notes.as_deref(), Some("Router admin")); +} + +#[test] +fn csv_generic_bom_crlf_extra_columns() { + let p = parse(include_bytes!("fixtures/generic-bom-crlf.csv"), &Format::Csv, NOW).unwrap(); + assert_eq!(p.items.len(), 2); + let mail = by_title(&p, "Mail"); + assert_eq!(mail.urls, vec!["mail.example.com"]); + assert_eq!(mail.username.as_deref(), Some("alex")); + assert_eq!(mail.totp.as_deref(), Some("JBSWY3DPEHPK3PXP")); + assert_eq!( + mail.notes.as_deref(), + Some("multi\r\nline\n\n— Imported fields —\nSecurity Question: Synthetic Fluffy") + ); + // No title: falls back to the URL host. + let bank = by_title(&p, "bank.example.com"); + assert_eq!(bank.password.as_deref(), Some("bank-synthetic-pw")); +} + +#[test] +fn csv_ragged_rows() { + let p = parse(b"title,password\nA,1,surplus\nB\n", &Format::Csv, NOW).unwrap(); + assert_eq!(by_title(&p, "A").notes.as_deref(), Some("— Imported fields —\nColumn 3: surplus")); + assert!(by_title(&p, "B").password.is_none()); +} + +#[test] +fn encrypted_export_round_trip() { + let source = parse(&fixture_1pux(), &Format::OnePux, NOW).unwrap(); + let mut items = source.items.clone(); + items[0].passkeys.push(kv_model::Passkey { + credential_id: vec![1, 2, 3], + rp_id: "example.com".into(), + user_handle: vec![4], + user_name: "alex".into(), + private_key: vec![5; 32], + sign_count: 7, + created_at: 1, + }); + items[0].android_apps.push("com.example.git".into()); + let blob = export::encrypt(&items, "export passphrase 1", NOW).unwrap(); + assert!(export::looks_like_export(&blob)); + assert_eq!(&blob[..4], b"KVEX"); + // Ciphertext does not contain plaintext. + assert!(!blob.windows(12).any(|w| w == b"correct-hors")); + + let back = parse(&blob, &Format::Encrypted { passphrase: " export passphrase 1 ".into() }, NOW).unwrap(); + assert_eq!(back.items.len(), items.len()); + for (a, b) in items.iter().zip(&back.items) { + assert_ne!(a.id, b.id, "restored items get fresh ids"); + let mut b = b.clone(); + b.id = a.id; + assert_eq!(*a, b); + } + + assert_eq!( + parse(&blob, &Format::Encrypted { passphrase: "wrong passphrase".into() }, NOW).unwrap_err(), + ImportError::BadPassphrase + ); + // Header is authenticated: lowering t (or any header byte) breaks decryption or is refused. + let mut tampered = blob.clone(); + tampered[10] = 4; + assert_eq!( + parse(&tampered, &Format::Encrypted { passphrase: "export passphrase 1".into() }, NOW).unwrap_err(), + ImportError::BadPassphrase + ); + let mut weak = blob.clone(); + weak[10] = 1; + assert!(matches!( + parse(&weak, &Format::Encrypted { passphrase: "export passphrase 1".into() }, NOW), + Err(ImportError::Unsupported(_)) + )); + let mut future = blob; + future[4] = 2; + assert!(matches!( + parse(&future, &Format::Encrypted { passphrase: "export passphrase 1".into() }, NOW), + Err(ImportError::Unsupported(_)) + )); +}