kv-import: 1Password .1pux and CSV import, encrypted export (M5)

New crate kv-import (pure parsing, no store) and kv-ffi/src/import.rs: - .1pux: accounts -> vaults -> items; Login/Password/Card/Note/Identity mapped, other categories as secure notes; unmapped fields appended to notes; attachments and password history reported. - CSV: 1Password 8 export and generic headers (aliases, BOM, quotes, embedded newlines, extra columns into notes). - Duplicates on (title, username, URL host): skip or add " (imported)". - Encrypted export KVEX v1: Argon2id (>= account KDF minimum) + XChaCha20-Poly1305, authenticated header, padded JSON; restorable through import. - Caps: 100 MB input, 64 MB decompressed export.data, 10k zip entries, 100k items. - FFI: VaultCore.import_preview / import_commit / export_encrypted (unlocked only); items are sealed, seq-stamped and queued for sync like save_item. Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-17 00:36:21 +09:00
parent 5eb2b622dd
commit 26664e141b
15 changed files with 2052 additions and 0 deletions
@@ -0,0 +1,316 @@
+//! Fixture-based tests. All fixture data is synthetic (example.com domains, test card numbers).
+
+use std::io::{Cursor, Write};
+
+use kv_import::{export, parse, parse_with_limits, Format, ImportError, Limits, Parsed};
+use kv_model::{Item, ItemKind};
+use zip::write::SimpleFileOptions;
+
+const NOW: i64 = 1_760_000_000;
+const EXPORT_DATA: &str = include_str!("fixtures/1pux-export.data.json");
+
+/// Builds a .1pux the way 1Password lays it out.
+fn onepux(entries: &[(&str, &[u8])]) -> Vec<u8> {
+    let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));
+    let opts = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated);
+    for (name, data) in entries {
+        if name.ends_with('/') {
+            zip.add_directory(*name, opts).unwrap();
+        } else {
+            zip.start_file(*name, opts).unwrap();
+            zip.write_all(data).unwrap();
+        }
+    }
+    zip.finish().unwrap().into_inner()
+}
+
+fn fixture_1pux() -> Vec<u8> {
+    onepux(&[
+        ("export.attributes", br#"{"version":3,"description":"1Password Unencrypted Export","createdAt":1700000000}"#),
+        ("export.data", EXPORT_DATA.as_bytes()),
+        ("files/", b""),
+        ("files/doc0000000000000000000000a__policy.pdf", b"%PDF-1.4 synthetic"),
+    ])
+}
+
+fn by_title<'a>(p: &'a Parsed, title: &str) -> &'a Item {
+    p.items.iter().find(|i| i.title == title).unwrap_or_else(|| panic!("no item {title}"))
+}
+
+#[test]
+fn onepux_fixture() {
+    let p = parse(&fixture_1pux(), &Format::OnePux, NOW).unwrap();
+    // 9 items, one deleted.
+    assert_eq!(p.items.len(), 8);
+    assert_eq!(p.count(ItemKind::Login), 4);
+    assert_eq!(p.count(ItemKind::Card), 1);
+    assert_eq!(p.count(ItemKind::Identity), 1);
+    assert_eq!(p.count(ItemKind::SecureNote), 2);
+
+    let git = by_title(&p, "Example Git");
+    assert_eq!(git.kind, ItemKind::Login);
+    assert_eq!(git.username.as_deref(), Some("[email protected]"));
+    assert_eq!(git.password.as_deref(), Some("correct-horse-battery-staple"));
+    assert_eq!(git.urls, vec!["https://git.example.com/login", "https://www.example.com"]);
+    assert!(git.totp.as_deref().unwrap().starts_with("otpauth://totp/Example:"));
+    assert!(git.favorite);
+    assert_eq!(git.tags, vec!["dev", "work", "Personal"]);
+    assert_eq!(git.created_at, 1_614_298_956);
+    assert_eq!(git.modified_at, 1_700_000_000);
+    assert_eq!(git.seq, 1);
+    let notes = git.notes.as_deref().unwrap();
+    assert!(notes.starts_with("Main code hosting account.\n\n— Imported from 1Password —"));
+    assert!(notes.contains("[Security questions]\nFirst pet: Synthetic Fluffy\nRecovery codes: 1111-2222\n3333-4444"));
+
+    let forum = by_title(&p, "Old Forum");
+    assert_eq!(forum.password.as_deref(), Some(" leading-space-pw"), "passwords are kept verbatim");
+    assert_eq!(forum.totp.as_deref(), Some("GEZDGNBVGY3TQOJQ"));
+    assert!(forum.tags.contains(&"Archived".to_string()));
+    assert!(forum.notes.is_none());
+
+    let card = by_title(&p, "Example Visa");
+    assert_eq!(card.kind, ItemKind::Card);
+    let n = card.notes.as_deref().unwrap();
+    for line in [
+        "Travel card.",
+        "cardholder name: ALEX EXAMPLE",
+        "type: visa",
+        "number: 4111111111111111",
+        "verification number: 123",
+        "expiry date: 12/2027",
+        "[Contact Information]\nissuing bank: Example Bank",
+    ] {
+        assert!(n.contains(line), "card notes missing {line:?}:\n{n}");
+    }
+
+    let note = by_title(&p, "Home Wi-Fi");
+    assert_eq!(note.kind, ItemKind::SecureNote);
+    assert_eq!(note.notes.as_deref(), Some("Wi-Fi: SyntheticNet\nKey: not-a-real-key"));
+
+    let id = by_title(&p, "Alex Example");
+    assert_eq!(id.kind, ItemKind::Identity);
+    let n = id.notes.as_deref().unwrap();
+    assert!(n.contains("first name: Alex"));
+    assert!(n.contains("birth date: 2000-02-29"));
+    assert!(n.contains("address: 1 Example Way, Sampletown, CA, 00000, us"));
+    assert!(n.contains("email: [email protected]"));
+
+    let router = by_title(&p, "Router");
+    assert_eq!(router.kind, ItemKind::Login);
+    assert_eq!(router.password.as_deref(), Some("router-synthetic-pw"));
+
+    let server = by_title(&p, "Build Server");
+    assert_eq!(server.kind, ItemKind::SecureNote);
+    assert!(server.notes.as_deref().unwrap().contains("password: server-synthetic-pw"));
+    assert_eq!(server.tags, vec!["Shared"]);
+
+    // The attachment is reported, not imported.
+    assert_eq!(p.attachments_skipped, 1);
+    assert!(p.warnings.iter().any(|w| w.contains("\"Example Insurance\": attachment \"policy.pdf\" was not imported")));
+    assert!(p.warnings.iter().any(|w| w.contains("1 previous password")));
+    assert!(p.warnings.iter().any(|w| w.contains("1 deleted item")));
+    assert!(p.warnings.iter().any(|w| w.contains("1 item(s) of other categories")));
+    // Warnings never carry secret values.
+    for w in &p.warnings {
+        for secret in ["correct-horse", "4111", "server-synthetic-pw", "old-synthetic-password"] {
+            assert!(!w.contains(secret), "warning leaks a secret: {w}");
+        }
+    }
+}
+
+#[test]
+fn onepux_unreferenced_files_are_reported() {
+    let data = br#"{"accounts":[{"attrs":{},"vaults":[{"attrs":{"name":"Personal"},"items":[]}]}]}"#;
+    let zip = onepux(&[("export.data", data), ("files/x__a.bin", b"1"), ("files/y__b.bin", b"2")]);
+    let p = parse(&zip, &Format::OnePux, NOW).unwrap();
+    assert!(p.items.is_empty());
+    assert_eq!(p.attachments_skipped, 2);
+}
+
+#[test]
+fn zip_bomb_decompressed_size() {
+    // 8 MiB of zeros compresses to a few KiB.
+    let zeros = vec![b' '; 8 * 1024 * 1024];
+    let zip = onepux(&[("export.data", &zeros)]);
+    assert!(zip.len() < 100 * 1024);
+    let limits = Limits { max_decompressed: 1024 * 1024, ..Limits::default() };
+    assert_eq!(
+        parse_with_limits(&zip, &Format::OnePux, NOW, &limits).unwrap_err(),
+        ImportError::TooLarge { limit: 1024 * 1024 }
+    );
+}
+
+#[test]
+fn zip_bomb_entry_count() {
+    let names: Vec<String> = (0..50).map(|i| format!("files/{i}__f")).collect();
+    let mut entries: Vec<(&str, &[u8])> = names.iter().map(|n| (n.as_str(), &b""[..])).collect();
+    entries.push(("export.data", b"{\"accounts\":[]}"));
+    let limits = Limits { max_entries: 20, ..Limits::default() };
+    assert_eq!(
+        parse_with_limits(&onepux(&entries), &Format::OnePux, NOW, &limits).unwrap_err(),
+        ImportError::TooManyEntries { limit: 20 }
+    );
+}
+
+#[test]
+fn input_size_cap() {
+    let limits = Limits { max_input: 10, ..Limits::default() };
+    assert_eq!(
+        parse_with_limits(&[0u8; 11], &Format::Csv, NOW, &limits).unwrap_err(),
+        ImportError::TooLarge { limit: 10 }
+    );
+}
+
+#[test]
+fn item_count_cap() {
+    let csv = "title,password\na,1\nb,2\nc,3\n";
+    let limits = Limits { max_items: 2, ..Limits::default() };
+    assert!(matches!(parse_with_limits(csv.as_bytes(), &Format::Csv, NOW, &limits), Err(ImportError::Malformed(_))));
+}
+
+#[test]
+fn malformed_inputs_error_without_panicking() {
+    let full = fixture_1pux();
+    let cases: Vec<(Vec<u8>, &str)> = vec![
+        (vec![], "empty"),
+        (b"not a zip at all".to_vec(), "garbage"),
+        (full[..full.len() / 2].to_vec(), "truncated zip"),
+        (onepux(&[("other.json", b"{}")]), "no export.data"),
+        (onepux(&[("export.data", b"{not json")]), "bad json"),
+        (onepux(&[("export.data", b"[1,2,3]")]), "no accounts"),
+        (onepux(&[("export.data", br#"{"accounts":[{"vaults":[{"items":[42]}]}]}"#)]), "item not object"),
+        (
+            onepux(&[(
+                "export.data",
+                br#"{"accounts":[{"vaults":[{"items":[{"categoryUuid":7,"overview":[],"details":{"sections":"x","loginFields":[1,{"value":5}]}}]}]}]}"#,
+            )]),
+            "wrong types",
+        ),
+    ];
+    for (bytes, what) in cases {
+        let r = std::panic::catch_unwind(|| parse(&bytes, &Format::OnePux, NOW));
+        let r = r.unwrap_or_else(|_| panic!("{what}: panicked"));
+        if what == "wrong types" {
+            // Tolerated: odd field types are ignored, the item is still imported.
+            assert_eq!(r.unwrap().items.len(), 1, "{what}");
+        } else {
+            assert!(r.is_err(), "{what}: expected an error");
+        }
+    }
+
+    for (bytes, what) in [
+        (&b""[..], "empty csv"),
+        (b"\xff\xfe", "not utf-8"),
+        (b"foo,bar\n1,2\n", "unknown header"),
+        (b"title,password\n\"open,1\n", "unterminated quote"),
+    ] {
+        let r = std::panic::catch_unwind(|| parse(bytes, &Format::Csv, NOW)).unwrap_or_else(|_| panic!("{what}"));
+        assert!(r.is_err(), "{what}");
+    }
+
+    for (bytes, what) in [(&b"KVEX"[..], "short export"), (&[0u8; 200][..], "no magic")] {
+        let r = parse(bytes, &Format::Encrypted { passphrase: "a long passphrase".into() }, NOW);
+        assert!(matches!(r, Err(ImportError::Malformed(_))), "{what}");
+    }
+}
+
+#[test]
+fn csv_1password() {
+    let p = parse(include_bytes!("fixtures/1password8.csv"), &Format::Csv, NOW).unwrap();
+    assert_eq!(p.items.len(), 3);
+    let git = by_title(&p, "Example Git");
+    assert_eq!(git.username.as_deref(), Some("[email protected]"));
+    assert_eq!(git.password.as_deref(), Some("correct-horse-battery-staple"));
+    assert_eq!(git.urls, vec!["https://git.example.com/login"]);
+    assert!(git.totp.as_deref().unwrap().contains("secret=JBSWY3DPEHPK3PXP"));
+    assert!(git.favorite);
+    assert_eq!(git.tags, vec!["dev", "work"]);
+    assert_eq!(git.notes.as_deref(), Some("Main code hosting account.\nSecond line, with a comma and \"quotes\"."));
+
+    let forum = by_title(&p, "Old Forum");
+    assert_eq!(forum.password.as_deref(), Some(" leading-space-pw"));
+    assert_eq!(forum.tags, vec!["Archived"]);
+    assert!(p.warnings.iter().any(|w| w.contains("1 archived")));
+
+    let router = by_title(&p, "Router");
+    assert!(router.urls.is_empty());
+    assert_eq!(router.notes.as_deref(), Some("Router admin"));
+}
+
+#[test]
+fn csv_generic_bom_crlf_extra_columns() {
+    let p = parse(include_bytes!("fixtures/generic-bom-crlf.csv"), &Format::Csv, NOW).unwrap();
+    assert_eq!(p.items.len(), 2);
+    let mail = by_title(&p, "Mail");
+    assert_eq!(mail.urls, vec!["mail.example.com"]);
+    assert_eq!(mail.username.as_deref(), Some("alex"));
+    assert_eq!(mail.totp.as_deref(), Some("JBSWY3DPEHPK3PXP"));
+    assert_eq!(
+        mail.notes.as_deref(),
+        Some("multi\r\nline\n\n— Imported fields —\nSecurity Question: Synthetic Fluffy")
+    );
+    // No title: falls back to the URL host.
+    let bank = by_title(&p, "bank.example.com");
+    assert_eq!(bank.password.as_deref(), Some("bank-synthetic-pw"));
+}
+
+#[test]
+fn csv_ragged_rows() {
+    let p = parse(b"title,password\nA,1,surplus\nB\n", &Format::Csv, NOW).unwrap();
+    assert_eq!(by_title(&p, "A").notes.as_deref(), Some("— Imported fields —\nColumn 3: surplus"));
+    assert!(by_title(&p, "B").password.is_none());
+}
+
+#[test]
+fn encrypted_export_round_trip() {
+    let source = parse(&fixture_1pux(), &Format::OnePux, NOW).unwrap();
+    let mut items = source.items.clone();
+    items[0].passkeys.push(kv_model::Passkey {
+        credential_id: vec![1, 2, 3],
+        rp_id: "example.com".into(),
+        user_handle: vec![4],
+        user_name: "alex".into(),
+        private_key: vec![5; 32],
+        sign_count: 7,
+        created_at: 1,
+    });
+    items[0].android_apps.push("com.example.git".into());
+    let blob = export::encrypt(&items, "export passphrase 1", NOW).unwrap();
+    assert!(export::looks_like_export(&blob));
+    assert_eq!(&blob[..4], b"KVEX");
+    // Ciphertext does not contain plaintext.
+    assert!(!blob.windows(12).any(|w| w == b"correct-hors"));
+
+    let back = parse(&blob, &Format::Encrypted { passphrase: "  export passphrase 1 ".into() }, NOW).unwrap();
+    assert_eq!(back.items.len(), items.len());
+    for (a, b) in items.iter().zip(&back.items) {
+        assert_ne!(a.id, b.id, "restored items get fresh ids");
+        let mut b = b.clone();
+        b.id = a.id;
+        assert_eq!(*a, b);
+    }
+
+    assert_eq!(
+        parse(&blob, &Format::Encrypted { passphrase: "wrong passphrase".into() }, NOW).unwrap_err(),
+        ImportError::BadPassphrase
+    );
+    // Header is authenticated: lowering t (or any header byte) breaks decryption or is refused.
+    let mut tampered = blob.clone();
+    tampered[10] = 4;
+    assert_eq!(
+        parse(&tampered, &Format::Encrypted { passphrase: "export passphrase 1".into() }, NOW).unwrap_err(),
+        ImportError::BadPassphrase
+    );
+    let mut weak = blob.clone();
+    weak[10] = 1;
+    assert!(matches!(
+        parse(&weak, &Format::Encrypted { passphrase: "export passphrase 1".into() }, NOW),
+        Err(ImportError::Unsupported(_))
+    ));
+    let mut future = blob;
+    future[4] = 2;
+    assert!(matches!(
+        parse(&future, &Format::Encrypted { passphrase: "export passphrase 1".into() }, NOW),
+        Err(ImportError::Unsupported(_))
+    ));
+}