kv-import: 1Password .1pux and CSV import, encrypted export (M5)
New crate kv-import (pure parsing, no store) and kv-ffi/src/import.rs: - .1pux: accounts -> vaults -> items; Login/Password/Card/Note/Identity mapped, other categories as secure notes; unmapped fields appended to notes; attachments and password history reported. - CSV: 1Password 8 export and generic headers (aliases, BOM, quotes, embedded newlines, extra columns into notes). - Duplicates on (title, username, URL host): skip or add " (imported)". - Encrypted export KVEX v1: Argon2id (>= account KDF minimum) + XChaCha20-Poly1305, authenticated header, padded JSON; restorable through import. - Caps: 100 MB input, 64 MB decompressed export.data, 10k zip entries, 100k items. - FFI: VaultCore.import_preview / import_commit / export_encrypted (unlocked only); items are sealed, seq-stamped and queued for sync like save_item. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,297 @@
|
||||
//! 1Password Unencrypted Export (`.1pux`): a ZIP holding `export.attributes`, `export.data`
|
||||
//! (JSON: `accounts[].vaults[].items[]`) and `files/` (attachments, not imported).
|
||||
//!
|
||||
//! Mapping (docs/09): Login 001 and Password 005 → login; Credit Card 002 → card; Secure Note 003
|
||||
//! → secure note; Identity 004 → identity; every other category → secure note. Fields without a
|
||||
//! home in `Item` (card and identity fields, custom sections, extra form fields) are appended to
|
||||
//! the notes as `Label: value` lines under a "— Imported from 1Password —" banner.
|
||||
|
||||
use std::io::{Cursor, Read};
|
||||
|
||||
use kv_model::{Item, ItemKind};
|
||||
use serde_json::Value;
|
||||
use zeroize::{Zeroize, Zeroizing};
|
||||
|
||||
use crate::{malformed, new_item, set_opt, Extras, ImportError, Limits, Parsed, Result};
|
||||
|
||||
const DATA_ENTRY: &str = "export.data";
|
||||
const BANNER: &str = "— Imported from 1Password —";
|
||||
|
||||
/// JSON whose strings are wiped on drop.
|
||||
struct Wiped(Value);
|
||||
|
||||
impl Drop for Wiped {
|
||||
fn drop(&mut self) {
|
||||
wipe(&mut self.0);
|
||||
}
|
||||
}
|
||||
|
||||
fn wipe(v: &mut Value) {
|
||||
match v {
|
||||
Value::String(s) => s.zeroize(),
|
||||
Value::Array(a) => a.iter_mut().for_each(wipe),
|
||||
Value::Object(m) => m.values_mut().for_each(wipe),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads `export.data` with the entry-count and decompressed-size caps.
|
||||
pub fn read_export_data(bytes: &[u8], limits: &Limits) -> Result<(Zeroizing<Vec<u8>>, usize)> {
|
||||
let mut zip =
|
||||
zip::ZipArchive::new(Cursor::new(bytes)).map_err(|e| malformed(format!("not a .1pux archive: {e}")))?;
|
||||
if zip.len() > limits.max_entries {
|
||||
return Err(ImportError::TooManyEntries { limit: limits.max_entries });
|
||||
}
|
||||
let files =
|
||||
zip.file_names().filter_map(|n| n.ok()).filter(|n| n.starts_with("files/") && !n.ends_with('/')).count();
|
||||
let entry = zip.by_name(DATA_ENTRY).map_err(|_| malformed("export.data is missing"))?;
|
||||
if entry.size() > limits.max_decompressed {
|
||||
return Err(ImportError::TooLarge { limit: limits.max_decompressed });
|
||||
}
|
||||
// Never trust the declared size: stop reading one byte past the cap.
|
||||
let mut out = Zeroizing::new(Vec::new());
|
||||
entry
|
||||
.take(limits.max_decompressed + 1)
|
||||
.read_to_end(&mut out)
|
||||
.map_err(|e| malformed(format!("export.data: {e}")))?;
|
||||
if out.len() as u64 > limits.max_decompressed {
|
||||
return Err(ImportError::TooLarge { limit: limits.max_decompressed });
|
||||
}
|
||||
Ok((out, files))
|
||||
}
|
||||
|
||||
pub fn parse(bytes: &[u8], now: i64, limits: &Limits) -> Result<Parsed> {
|
||||
let (data, files_in_archive) = read_export_data(bytes, limits)?;
|
||||
let root = Wiped(serde_json::from_slice(&data).map_err(|e| malformed(format!("export.data: {e}")))?);
|
||||
let accounts =
|
||||
root.0.get("accounts").and_then(Value::as_array).ok_or_else(|| malformed("export.data has no accounts"))?;
|
||||
|
||||
let vault_count: usize =
|
||||
accounts.iter().map(|a| a.get("vaults").and_then(Value::as_array).map_or(0, Vec::len)).sum();
|
||||
let mut ctx = Ctx { out: Parsed::default(), now, password_history: 0, other_category: 0, trashed: 0, archived: 0 };
|
||||
for account in accounts {
|
||||
for vault in account.get("vaults").and_then(Value::as_array).into_iter().flatten() {
|
||||
let vault_name = str_at(vault, &["attrs", "name"]);
|
||||
for raw in vault.get("items").and_then(Value::as_array).into_iter().flatten() {
|
||||
limits.check_items(ctx.out.items.len() + 1)?;
|
||||
if let Some(mut item) = ctx.item(raw)? {
|
||||
if vault_count > 1 && !vault_name.is_empty() {
|
||||
item.tags.push(vault_name.to_string());
|
||||
}
|
||||
ctx.out.items.push(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let Ctx { mut out, password_history, other_category, trashed, archived, .. } = ctx;
|
||||
if out.attachments_skipped == 0 && files_in_archive > 0 {
|
||||
out.attachments_skipped = files_in_archive as u32;
|
||||
out.warnings.push(format!(
|
||||
"{files_in_archive} file(s) in the archive were not imported (attachments are not supported yet)"
|
||||
));
|
||||
}
|
||||
if password_history > 0 {
|
||||
out.warnings.push(format!("{password_history} previous password(s) from password history were not imported"));
|
||||
}
|
||||
if other_category > 0 {
|
||||
out.warnings.push(format!("{other_category} item(s) of other categories were imported as secure notes"));
|
||||
}
|
||||
if archived > 0 {
|
||||
out.warnings.push(format!("{archived} archived item(s) were imported with the tag \"Archived\""));
|
||||
}
|
||||
if trashed > 0 {
|
||||
out.warnings.push(format!("{trashed} deleted item(s) were skipped"));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
struct Ctx {
|
||||
out: Parsed,
|
||||
now: i64,
|
||||
password_history: u32,
|
||||
other_category: u32,
|
||||
trashed: u32,
|
||||
archived: u32,
|
||||
}
|
||||
|
||||
impl Ctx {
|
||||
fn item(&mut self, raw: &Value) -> Result<Option<Item>> {
|
||||
if !raw.is_object() {
|
||||
return Err(malformed("item is not an object"));
|
||||
}
|
||||
let state = str_at(raw, &["state"]);
|
||||
if state == "deleted" || state == "trashed" {
|
||||
self.trashed += 1;
|
||||
return Ok(None);
|
||||
}
|
||||
let category = str_at(raw, &["categoryUuid"]);
|
||||
let kind = match category {
|
||||
"001" | "005" => ItemKind::Login,
|
||||
"002" => ItemKind::Card,
|
||||
"003" => ItemKind::SecureNote,
|
||||
"004" => ItemKind::Identity,
|
||||
_ => {
|
||||
self.other_category += 1;
|
||||
ItemKind::SecureNote
|
||||
}
|
||||
};
|
||||
let overview = raw.get("overview").unwrap_or(&Value::Null);
|
||||
let details = raw.get("details").unwrap_or(&Value::Null);
|
||||
let mut item = new_item(kind, str_at(overview, &["title"]), self.now);
|
||||
let title = item.title.clone();
|
||||
if let Some(t) = raw.get("createdAt").and_then(Value::as_i64) {
|
||||
item.created_at = t;
|
||||
}
|
||||
if let Some(t) = raw.get("updatedAt").and_then(Value::as_i64) {
|
||||
item.modified_at = t.max(item.created_at);
|
||||
}
|
||||
item.favorite = raw.get("favIndex").and_then(Value::as_i64).is_some_and(|f| f > 0);
|
||||
item.tags = overview
|
||||
.get("tags")
|
||||
.and_then(Value::as_array)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter_map(Value::as_str)
|
||||
.map(String::from)
|
||||
.collect();
|
||||
if state == "archived" {
|
||||
item.tags.push("Archived".into());
|
||||
self.archived += 1;
|
||||
}
|
||||
let mut urls: Vec<String> = overview
|
||||
.get("urls")
|
||||
.and_then(Value::as_array)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter_map(|u| u.get("url").and_then(Value::as_str))
|
||||
.filter(|u| !u.trim().is_empty())
|
||||
.map(String::from)
|
||||
.collect();
|
||||
let main_url = str_at(overview, &["url"]);
|
||||
if !main_url.is_empty() && !urls.iter().any(|u| u == main_url) {
|
||||
urls.insert(0, main_url.to_string());
|
||||
}
|
||||
item.urls = urls;
|
||||
set_opt(&mut item.notes, str_at(details, &["notesPlain"]));
|
||||
|
||||
let mut extras = Extras::default();
|
||||
if category == "005" {
|
||||
set_opt(&mut item.password, str_at(details, &["password"]));
|
||||
} else if let Some(pw) = details.get("password").and_then(Value::as_str) {
|
||||
extras.push("Password", pw);
|
||||
}
|
||||
for f in details.get("loginFields").and_then(Value::as_array).into_iter().flatten() {
|
||||
let value = str_at(f, &["value"]);
|
||||
match str_at(f, &["designation"]) {
|
||||
"username" if item.username.is_none() && kind == ItemKind::Login => set_opt(&mut item.username, value),
|
||||
"password" if item.password.is_none() && kind == ItemKind::Login => set_opt(&mut item.password, value),
|
||||
_ => {
|
||||
let name = str_at(f, &["name"]);
|
||||
extras.push(if name.is_empty() { "Form field" } else { name }, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
for section in details.get("sections").and_then(Value::as_array).into_iter().flatten() {
|
||||
let section_title = str_at(section, &["title"]);
|
||||
extras.heading(section_title);
|
||||
for field in section.get("fields").and_then(Value::as_array).into_iter().flatten() {
|
||||
self.field(&mut item, &mut extras, &title, field);
|
||||
}
|
||||
}
|
||||
if let Some(doc) = details.get("documentAttributes").filter(|d| d.is_object()) {
|
||||
self.attachment(&title, doc);
|
||||
}
|
||||
self.password_history += details.get("passwordHistory").and_then(Value::as_array).map_or(0, |h| h.len() as u32);
|
||||
extras.append_to(&mut item, BANNER);
|
||||
Ok(Some(item))
|
||||
}
|
||||
|
||||
fn field(&mut self, item: &mut Item, extras: &mut Extras, title: &str, field: &Value) {
|
||||
let label = {
|
||||
let t = str_at(field, &["title"]);
|
||||
if t.is_empty() {
|
||||
str_at(field, &["id"])
|
||||
} else {
|
||||
t
|
||||
}
|
||||
};
|
||||
let Some(value) = field.get("value").and_then(Value::as_object) else { return };
|
||||
for (vtype, v) in value {
|
||||
match vtype.as_str() {
|
||||
"totp" if item.totp.is_none() && v.as_str().is_some_and(|s| !s.trim().is_empty()) => {
|
||||
item.totp = v.as_str().map(|s| s.trim().to_string());
|
||||
}
|
||||
"file" => self.attachment(title, v),
|
||||
_ => {
|
||||
let text = Zeroizing::new(render(vtype, v));
|
||||
extras.push(label, &text);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn attachment(&mut self, title: &str, attrs: &Value) {
|
||||
self.out.attachments_skipped += 1;
|
||||
let name = str_at(attrs, &["fileName"]);
|
||||
self.out.warnings.push(format!(
|
||||
"\"{title}\": attachment \"{}\" was not imported (attachments are not supported yet)",
|
||||
if name.is_empty() { "unnamed" } else { name }
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
/// Text for a 1Password field value (`{"<type>": <value>}`).
|
||||
fn render(vtype: &str, v: &Value) -> String {
|
||||
match (vtype, v) {
|
||||
(_, Value::String(s)) => s.clone(),
|
||||
("date", Value::Number(n)) => n.as_i64().map(format_date).unwrap_or_else(|| n.to_string()),
|
||||
("monthYear", Value::Number(n)) => match n.as_i64() {
|
||||
Some(ym) if ym > 0 => format!("{:02}/{}", ym % 100, ym / 100),
|
||||
_ => n.to_string(),
|
||||
},
|
||||
("email", Value::Object(o)) => o.get("email_address").and_then(Value::as_str).unwrap_or("").to_string(),
|
||||
("address", Value::Object(o)) => ["street", "city", "state", "zip", "country"]
|
||||
.iter()
|
||||
.filter_map(|k| o.get(*k).and_then(Value::as_str))
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", "),
|
||||
("sshKey", Value::Object(o)) => o.get("privateKey").and_then(Value::as_str).unwrap_or("").to_string(),
|
||||
(_, Value::Null) => String::new(),
|
||||
(_, Value::Bool(b)) => b.to_string(),
|
||||
(_, Value::Number(n)) => n.to_string(),
|
||||
// Unknown structured value: keep it verbatim rather than drop it.
|
||||
(_, other) => other.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// `YYYY-MM-DD` (UTC) for a Unix timestamp in seconds.
|
||||
fn format_date(secs: i64) -> String {
|
||||
let z = secs.div_euclid(86_400) + 719_468;
|
||||
let era = z.div_euclid(146_097);
|
||||
let doe = z.rem_euclid(146_097);
|
||||
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
|
||||
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
|
||||
let mp = (5 * doy + 2) / 153;
|
||||
let d = doy - (153 * mp + 2) / 5 + 1;
|
||||
let m = if mp < 10 { mp + 3 } else { mp - 9 };
|
||||
let y = yoe + era * 400 + i64::from(m <= 2);
|
||||
format!("{y:04}-{m:02}-{d:02}")
|
||||
}
|
||||
|
||||
fn str_at<'a>(v: &'a Value, path: &[&str]) -> &'a str {
|
||||
path.iter().try_fold(v, |v, k| v.get(k)).and_then(Value::as_str).unwrap_or("")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn dates() {
|
||||
assert_eq!(format_date(0), "1970-01-01");
|
||||
assert_eq!(format_date(951_782_400), "2000-02-29");
|
||||
assert_eq!(format_date(-86_400), "1969-12-31");
|
||||
assert_eq!(render("monthYear", &serde_json::json!(202_712)), "12/2027");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user