diff --git a/src/database/custom_3dmv.rs b/src/database/custom_3dmv.rs index a5379bd..801fca0 100644 --- a/src/database/custom_3dmv.rs +++ b/src/database/custom_3dmv.rs @@ -53,12 +53,18 @@ pub fn next_mv_id() -> i64 { std::cmp::max(std::cmp::max(issued, max), FIRST_MV_ID - 1) + 1 } -pub fn insert_mv(mv_id: i64, music_id: i64, owner_id: i64, mv: &JsonValue, published: bool) { - DATABASE.lock_and_exec( - "INSERT INTO mvs (mv_id, music_id, owner_id, mv, published) VALUES (?1, ?2, ?3, ?4, ?5)", - params!(mv_id, music_id, owner_id, jzon::stringify(mv.clone()), published as i64) - ); - DATABASE.lock_and_exec("INSERT INTO revision (id, revision, last_mv_id) VALUES (1, 0, ?1) ON CONFLICT(id) DO UPDATE SET last_mv_id=?1", params!(mv_id)); +// The row and the high-water mark are one write: a failure between them leaves an +// MV whose id the next upload would reissue, and the failure has to reach the +// uploader as an error rather than panic the worker (lock_and_exec unwraps) +pub fn insert_mv(mv_id: i64, music_id: i64, owner_id: i64, mv: &JsonValue, published: bool) -> Result<(), rusqlite::Error> { + DATABASE.lock_and_transact(|conn| { + conn.execute( + "INSERT INTO mvs (mv_id, music_id, owner_id, mv, published) VALUES (?1, ?2, ?3, ?4, ?5)", + params!(mv_id, music_id, owner_id, jzon::stringify(mv.clone()), published as i64) + )?; + conn.execute("INSERT INTO revision (id, revision, last_mv_id) VALUES (1, 0, ?1) ON CONFLICT(id) DO UPDATE SET last_mv_id=?1", params!(mv_id))?; + Ok(()) + }) } // The catalog blob only. The owner and the published flag live in their own @@ -195,17 +201,24 @@ pub fn all_mv_blobs() -> Option { // Blobs are content-addressed and may be shared between MVs (or roles), so // every candidate row is checked - a single-row scan could land on a -// coincidental substring match and miss the real reference -pub fn blob_in_use(md5: &str) -> bool { - let rows = DATABASE.lock_and_select_all("SELECT mv FROM mvs WHERE mv LIKE ?1", params!(format!("%{}%", md5))).unwrap_or(array![]); +// coincidental substring match and miss the real reference. +// +// Returns Result and never folds an error into "not referenced": the caller +// unlinks on false, and a read that failed (SQLITE_BUSY under a concurrent +// write, a corrupt page) says nothing about whether the blob is live. The +// startup sweep is deliberately fail-closed; this is its online half +pub fn blob_in_use(md5: &str) -> Result { + let rows = DATABASE.lock_and_select_all("SELECT mv FROM mvs WHERE mv LIKE ?1", params!(format!("%{}%", md5)))?; for blob in rows.members() { - if let Ok(mv) = jzon::parse(&blob.to_string()) { - if mv["files"].members().any(|file| file["md5"].as_str() == Some(md5)) { - return true; - } + let Ok(mv) = jzon::parse(&blob.to_string()) else { + // An unparseable row could reference anything - assume it does + return Ok(true); + }; + if mv["files"].members().any(|file| file["md5"].as_str() == Some(md5)) { + return Ok(true); } } - false + Ok(false) } // Two-step content-addressed lookup for the data route: the LIKE scan finds a @@ -213,5 +226,38 @@ pub fn blob_in_use(md5: &str) -> bool { // stored file of a live MV (and not a substring coincidence elsewhere in the // blob). The blob path itself derives from the md5 pub fn find_blob_by_md5(md5: &str) -> bool { - blob_in_use(md5) + // A read failure here means "cannot prove this blob is served", which the + // route turns into a 404 - the client re-requests. Unlike the GC, guessing + // wrong in this direction costs nothing permanent + blob_in_use(md5).unwrap_or(false) +} + +// Every MV this account uploaded, for the account-deletion purge +pub fn mv_ids_for_owner(owner_id: i64) -> Vec { + let rows = DATABASE.lock_and_select_all("SELECT mv_id FROM mvs WHERE owner_id=?1 ORDER BY mv_id", params!(owner_id)).unwrap_or(array![]); + rows.members().filter_map(|id| id.as_i64()).collect() +} + +// The stored bytes an MV's files account for. Blobs are shared, so two MVs that +// carry the same model both count it: the quota is "what this account asked the +// server to store", not a dedup-aware disk figure +pub fn mv_bytes(files: &JsonValue) -> i64 { + files.members().map(|file| file["size"].as_i64().unwrap_or(0)).sum() +} + +// What this account's MVs already occupy, optionally ignoring one (the MV being +// replaced by an in-place edit, whose new size is counted instead) +pub fn owner_bytes(owner_id: i64, excluding_mv_id: i64) -> i64 { + let rows = parse_blobs(DATABASE.lock_and_select_all("SELECT mv FROM mvs WHERE owner_id=?1", params!(owner_id)).unwrap_or(array![])); + rows.members() + .filter(|mv| mv["mv_id"].as_i64() != Some(excluding_mv_id)) + .map(|mv| mv_bytes(&mv["files"])) + .sum() +} + +// The on-disk database file, so a test can force the read errors the fail-closed +// GC paths are built for +#[cfg(test)] +pub fn test_db_path() -> String { + DATABASE.get_path().to_string() } diff --git a/src/database/custom_card.rs b/src/database/custom_card.rs index 1714a37..b32a05f 100644 --- a/src/database/custom_card.rs +++ b/src/database/custom_card.rs @@ -87,20 +87,29 @@ pub fn next_character_id() -> i64 { std::cmp::max(std::cmp::max(issued, max), FIRST_CHARACTER_ID - 1) + 1 } -pub fn insert_card(master_card_id: i64, master_character_id: i64, owner_id: i64, card: &JsonValue, published: bool, obtainable: bool) { - DATABASE.lock_and_exec( - "INSERT INTO cards (master_card_id, master_character_id, owner_id, card, rarity, published, obtainable) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", - params!(master_card_id, master_character_id, owner_id, jzon::stringify(card.clone()), card["rarity"].as_i64().unwrap_or(1), published as i64, obtainable as i64) - ); - DATABASE.lock_and_exec("INSERT INTO revision (id, revision, last_card_id, last_character_id) VALUES (1, 0, ?1, 0) ON CONFLICT(id) DO UPDATE SET last_card_id=?1", params!(master_card_id)); +// The row and the high-water mark are one write: a failure between them leaves a +// card whose id the next upload would reissue, and the failure has to reach the +// uploader as an error rather than panic the worker (lock_and_exec unwraps) +pub fn insert_card(master_card_id: i64, master_character_id: i64, owner_id: i64, card: &JsonValue, published: bool, obtainable: bool) -> Result<(), rusqlite::Error> { + DATABASE.lock_and_transact(|conn| { + conn.execute( + "INSERT INTO cards (master_card_id, master_character_id, owner_id, card, rarity, published, obtainable) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + params!(master_card_id, master_character_id, owner_id, jzon::stringify(card.clone()), card["rarity"].as_i64().unwrap_or(1), published as i64, obtainable as i64) + )?; + conn.execute("INSERT INTO revision (id, revision, last_card_id, last_character_id) VALUES (1, 0, ?1, 0) ON CONFLICT(id) DO UPDATE SET last_card_id=?1", params!(master_card_id))?; + Ok(()) + }) } -pub fn insert_character(master_character_id: i64, owner_id: i64, character: &JsonValue) { - DATABASE.lock_and_exec( - "INSERT INTO characters (master_character_id, owner_id, character) VALUES (?1, ?2, ?3)", - params!(master_character_id, owner_id, jzon::stringify(character.clone())) - ); - DATABASE.lock_and_exec("INSERT INTO revision (id, revision, last_card_id, last_character_id) VALUES (1, 0, 0, ?1) ON CONFLICT(id) DO UPDATE SET last_character_id=?1", params!(master_character_id)); +pub fn insert_character(master_character_id: i64, owner_id: i64, character: &JsonValue) -> Result<(), rusqlite::Error> { + DATABASE.lock_and_transact(|conn| { + conn.execute( + "INSERT INTO characters (master_character_id, owner_id, character) VALUES (?1, ?2, ?3)", + params!(master_character_id, owner_id, jzon::stringify(character.clone())) + )?; + conn.execute("INSERT INTO revision (id, revision, last_card_id, last_character_id) VALUES (1, 0, 0, ?1) ON CONFLICT(id) DO UPDATE SET last_character_id=?1", params!(master_character_id))?; + Ok(()) + }) } // The catalog blob only. The owner and the published/obtainable flags live in @@ -173,6 +182,47 @@ pub fn has_character(master_character_id: i64) -> bool { DATABASE.lock_and_select("SELECT master_character_id FROM characters WHERE master_character_id=?1", params!(master_character_id)).is_ok() } +// Every card / character this account uploaded, for the account-deletion purge +pub fn card_ids_for_owner(owner_id: i64) -> Vec { + DATABASE.lock_and_select_all("SELECT master_card_id FROM cards WHERE owner_id=?1 ORDER BY master_card_id", params!(owner_id)) + .unwrap_or(array![]).members().filter_map(|id| id.as_i64()).collect() +} + +pub fn character_ids_for_owner(owner_id: i64) -> Vec { + DATABASE.lock_and_select_all("SELECT master_character_id FROM characters WHERE owner_id=?1 ORDER BY master_character_id", params!(owner_id)) + .unwrap_or(array![]).members().filter_map(|id| id.as_i64()).collect() +} + +// The stored bytes one card / character accounts for: every derived art png +// plus, for a character, its voiceline oggs +pub fn card_bytes(card: &JsonValue) -> i64 { + card["art"].members().map(|art| art["size"].as_i64().unwrap_or(0)).sum() +} + +pub fn character_bytes(character: &JsonValue) -> i64 { + let art: i64 = character["art"].members().map(|art| art["size"].as_i64().unwrap_or(0)).sum(); + let voice: i64 = character["voice"].members().map(|line| line["size"].as_i64().unwrap_or(0)).sum(); + art + voice +} + +// What this account's uploads already occupy. Cards and characters share one +// quota (they share one storage root), each optionally ignoring the entity being +// replaced by an in-place edit, whose new size is counted instead +pub fn owner_bytes(owner_id: i64, excluding_card_id: i64, excluding_character_id: i64) -> i64 { + let mut total = 0; + for blob in parse_blobs(DATABASE.lock_and_select_all("SELECT card FROM cards WHERE owner_id=?1", params!(owner_id)).unwrap_or(array![])).members() { + if blob["master_card_id"].as_i64() != Some(excluding_card_id) { + total += card_bytes(blob); + } + } + for blob in parse_blobs(DATABASE.lock_and_select_all("SELECT character FROM characters WHERE owner_id=?1", params!(owner_id)).unwrap_or(array![])).members() { + if blob["master_character_id"].as_i64() != Some(excluding_character_id) { + total += character_bytes(blob); + } + } + total +} + pub fn card_count_for_owner(owner_id: i64) -> i64 { DATABASE.lock_and_select_type::("SELECT COUNT(*) FROM cards WHERE owner_id=?1", params!(owner_id)).unwrap_or(0) } @@ -410,10 +460,10 @@ mod tests { let first = next_card_id(); assert!(first >= FIRST_CARD_ID); assert_ne!(first % 10000, 0); - insert_card(first, 1001, 3001, &card_blob(first, 1), false, false); + insert_card(first, 1001, 3001, &card_blob(first, 1), false, false).unwrap(); let second = next_card_id(); assert_eq!(second, first + 1); - insert_card(second, 1001, 3001, &card_blob(second, 1), false, false); + insert_card(second, 1001, 3001, &card_blob(second, 1), false, false).unwrap(); delete_card(second); assert_eq!(get_card(second), None); // The high-water mark survives the delete, so the id is retired @@ -421,7 +471,7 @@ mod tests { let character = next_character_id(); assert!(character >= FIRST_CHARACTER_ID); - insert_character(character, 3001, &object!{ "master_character_id": character }); + insert_character(character, 3001, &object!{ "master_character_id": character }).unwrap(); assert_eq!(next_character_id(), character + 1); delete_character(character); assert_eq!(next_character_id(), character + 1); @@ -440,13 +490,13 @@ mod tests { wipe(3004); let character = next_character_id(); - insert_character(character, 3003, &object!{ "master_character_id": character }); + insert_character(character, 3003, &object!{ "master_character_id": character }).unwrap(); let published = next_card_id(); let mut blob = card_blob(published, 3); blob["master_character_id"] = character.into(); - insert_card(published, character, 3003, &blob, true, true); + insert_card(published, character, 3003, &blob, true, true).unwrap(); let draft = next_card_id(); - insert_card(draft, character, 3003, &card_blob(draft, 1), false, false); + insert_card(draft, character, 3003, &card_blob(draft, 1), false, false).unwrap(); let owner_view = get_cards_for_user(3003, &[]); assert_eq!(owner_view.len(), 2); @@ -493,13 +543,13 @@ mod tests { wipe(3005); let r1 = next_card_id(); - insert_card(r1, 1001, 3005, &card_blob(r1, 1), true, true); + insert_card(r1, 1001, 3005, &card_blob(r1, 1), true, true).unwrap(); let r3 = next_card_id(); - insert_card(r3, 1001, 3005, &card_blob(r3, 3), true, true); + insert_card(r3, 1001, 3005, &card_blob(r3, 3), true, true).unwrap(); let unpublished = next_card_id(); - insert_card(unpublished, 1001, 3005, &card_blob(unpublished, 1), false, true); + insert_card(unpublished, 1001, 3005, &card_blob(unpublished, 1), false, true).unwrap(); let unobtainable = next_card_id(); - insert_card(unobtainable, 1001, 3005, &card_blob(unobtainable, 1), true, false); + insert_card(unobtainable, 1001, 3005, &card_blob(unobtainable, 1), true, false).unwrap(); assert_eq!(obtainable_card_ids(1), vec![r1]); assert_eq!(obtainable_card_ids(3), vec![r3]); @@ -516,7 +566,7 @@ mod tests { wipe(3007); let id = next_card_id(); - insert_card(id, 1001, 3007, &card_blob(id, 1), true, false); + insert_card(id, 1001, 3007, &card_blob(id, 1), true, false).unwrap(); assert_eq!(find_asset_by_md5(&format!("{:032x}", id)), Some(format!("{}/c_00.png", id))); assert_eq!(find_asset_by_md5("00000000000000000000000000000000"), None); @@ -524,7 +574,7 @@ mod tests { insert_character(character, 3007, &object!{ "master_character_id": character, "art": [{ "kind": "icon", "md5": "aabbccddeeff00112233445566778899", "size": 1 }] - }); + }).unwrap(); assert_eq!( find_asset_by_md5("aabbccddeeff00112233445566778899"), Some(format!("characters/{}/icon.png", character)) @@ -541,9 +591,9 @@ mod tests { wipe(3008); let alive = next_card_id(); - insert_card(alive, 1001, 3008, &card_blob(alive, 1), false, false); + insert_card(alive, 1001, 3008, &card_blob(alive, 1), false, false).unwrap(); let dead = next_card_id(); - insert_card(dead, 1001, 3008, &card_blob(dead, 1), true, false); + insert_card(dead, 1001, 3008, &card_blob(dead, 1), true, false).unwrap(); delete_card(dead); let dead_ids = dead_card_ids(&array![alive, dead, 10010001, 100010001, dead]); diff --git a/src/database/custom_song.rs b/src/database/custom_song.rs index 20db5f5..8b98d76 100644 --- a/src/database/custom_song.rs +++ b/src/database/custom_song.rs @@ -107,10 +107,16 @@ pub fn next_music_id() -> i64 { std::cmp::max(std::cmp::max(issued, max), FIRST_MUSIC_ID - 1) + 1 } -pub fn insert_song(music_id: i64, owner_id: i64, song: &JsonValue, visibility: &str, shared_with: &JsonValue, downloads_disabled: bool) { - DATABASE.lock_and_exec("INSERT INTO songs (music_id, owner_id, song, visibility, downloads_disabled) VALUES (?1, ?2, ?3, ?4, ?5)", params!(music_id, owner_id, jzon::stringify(song.clone()), visibility, downloads_disabled as i64)); - DATABASE.lock_and_exec("INSERT INTO revision (id, revision, last_music_id) VALUES (1, 0, ?1) ON CONFLICT(id) DO UPDATE SET last_music_id=?1", params!(music_id)); - set_shared_users(music_id, shared_with); +// The row, the high-water mark and the shared-user list are one write: a failure +// between them would leave a song whose id the next upload reissues, and the +// failure itself must reach the uploader as an error rather than panic the worker +// (lock_and_exec unwraps, lock_and_transact returns) +pub fn insert_song(music_id: i64, owner_id: i64, song: &JsonValue, visibility: &str, shared_with: &JsonValue, downloads_disabled: bool) -> Result<(), rusqlite::Error> { + DATABASE.lock_and_transact(|conn| { + conn.execute("INSERT INTO songs (music_id, owner_id, song, visibility, downloads_disabled) VALUES (?1, ?2, ?3, ?4, ?5)", params!(music_id, owner_id, jzon::stringify(song.clone()), visibility, downloads_disabled as i64))?; + conn.execute("INSERT INTO revision (id, revision, last_music_id) VALUES (1, 0, ?1) ON CONFLICT(id) DO UPDATE SET last_music_id=?1", params!(music_id))?; + write_shared_users(conn, music_id, shared_with) + }) } // Replace an existing song's catalog blob in place. The owner, visibility, @@ -119,9 +125,12 @@ pub fn update_song(music_id: i64, song: &JsonValue) { DATABASE.lock_and_exec("UPDATE songs SET song=?1 WHERE music_id=?2", params!(jzon::stringify(song.clone()), music_id)); } -pub fn delete_song(music_id: i64) { - DATABASE.lock_and_exec("DELETE FROM songs WHERE music_id=?1", params!(music_id)); - DATABASE.lock_and_exec("DELETE FROM shared_users WHERE music_id=?1", params!(music_id)); +pub fn delete_song(music_id: i64) -> Result<(), rusqlite::Error> { + DATABASE.lock_and_transact(|conn| { + conn.execute("DELETE FROM songs WHERE music_id=?1", params!(music_id))?; + conn.execute("DELETE FROM shared_users WHERE music_id=?1", params!(music_id))?; + Ok(()) + }) } pub fn get_song(music_id: i64) -> Option { @@ -133,9 +142,11 @@ pub fn get_song_owner(music_id: i64) -> Option { DATABASE.lock_and_select("SELECT owner_id FROM songs WHERE music_id=?1", params!(music_id)).ok()?.parse::().ok() } -pub fn set_visibility(music_id: i64, visibility: &str, shared_with: &JsonValue) { - DATABASE.lock_and_exec("UPDATE songs SET visibility=?1 WHERE music_id=?2", params!(visibility, music_id)); - set_shared_users(music_id, shared_with); +pub fn set_visibility(music_id: i64, visibility: &str, shared_with: &JsonValue) -> Result<(), rusqlite::Error> { + DATABASE.lock_and_transact(|conn| { + conn.execute("UPDATE songs SET visibility=?1 WHERE music_id=?2", params!(visibility, music_id))?; + write_shared_users(conn, music_id, shared_with) + }) } pub fn set_downloads_disabled(music_id: i64, downloads_disabled: bool) { @@ -146,11 +157,15 @@ fn get_downloads_disabled(music_id: i64) -> bool { DATABASE.lock_and_select("SELECT downloads_disabled FROM songs WHERE music_id=?1", params!(music_id)).unwrap_or_default() == "1" } -fn set_shared_users(music_id: i64, shared_with: &JsonValue) { - DATABASE.lock_and_exec("DELETE FROM shared_users WHERE music_id=?1", params!(music_id)); +// Replace-the-whole-list, inside the caller's transaction: a half-written list is a +// song shared with the wrong people +fn write_shared_users(conn: &rusqlite::Connection, music_id: i64, shared_with: &JsonValue) -> Result<(), rusqlite::Error> { + conn.execute("DELETE FROM shared_users WHERE music_id=?1", params!(music_id))?; for id in shared_with.members() { - DATABASE.lock_and_exec("INSERT OR IGNORE INTO shared_users (music_id, user_id) VALUES (?1, ?2)", params!(music_id, id.as_i64().unwrap())); + let Some(id) = id.as_i64() else { continue; }; + conn.execute("INSERT OR IGNORE INTO shared_users (music_id, user_id) VALUES (?1, ?2)", params!(music_id, id))?; } + Ok(()) } fn get_visibility(music_id: i64) -> String { @@ -257,6 +272,67 @@ pub fn public_song_title(music_id: i64, english: bool) -> Option { Some(if english && !name_en.is_empty() { name_en } else { name }) } +// Whether this viewer may fetch the song's per-id asset files (jacket, blur, +// chart JSON). Exactly the catalog's visibility rule: public to everyone, private +// to its owner, shared to its owner and the shared-user list. Anonymous viewers +// (no webui session) are only ever shown public songs. +// +// The /data/{md5} route stays sessionless on purpose - a 128-bit content hash IS +// the capability there - but /assets/{music_id}/{file} is addressed by a +// sequential id, so it needs the real rule +pub fn asset_visible(music_id: i64, viewer: Option) -> bool { + let Some(owner) = get_song_owner(music_id) else { + return false; + }; + let viewer = viewer.unwrap_or(0); + if owner == viewer { + return true; + } + match get_visibility(music_id).as_str() { + "public" => true, + "shared" => get_shared_users(music_id).contains(viewer), + _ => false + } +} + +// Every song this account uploaded, for the account-deletion purge +pub fn music_ids_by_owner(owner_id: i64) -> Vec { + DATABASE.lock_and_select_all("SELECT music_id FROM songs WHERE owner_id=?1 ORDER BY music_id", params!(owner_id)) + .unwrap_or(array![]) + .members().filter_map(|id| id.as_i64()).collect() +} + +// The served bytes one song accounts for: both jackets, every chart and both +// audio cues, straight out of the catalog blob that quotes them to the client. +// The original upload artifacts kept under original/ are NOT counted (the +// catalog does not record their sizes); the per-account cap is set with that +// roughly-2x on-disk factor in mind +pub fn song_bytes(song: &JsonValue) -> i64 { + let mut total = song["jacket_size"].as_i64().unwrap_or(0) + song["jacket_blur_size"].as_i64().unwrap_or(0); + for level in song["levels"].members() { + total += level["size"].as_i64().unwrap_or(0); + } + for key in ["play", "select"] { + total += song["sound"][key]["size"].as_i64().unwrap_or(0); + } + total +} + +// What this account's songs already occupy, optionally ignoring one song (the +// one being replaced by an in-place edit, whose new size is counted instead) +pub fn owner_bytes(owner_id: i64, excluding_music_id: i64) -> i64 { + let songs = DATABASE.lock_and_select_all("SELECT song FROM songs WHERE owner_id=?1", params!(owner_id)).unwrap_or(array![]); + let mut total = 0; + for blob in songs.members() { + let Ok(song) = jzon::parse(&blob.to_string()) else { continue; }; + if song["music_id"].as_i64() == Some(excluding_music_id) { + continue; + } + total += song_bytes(&song); + } + total +} + // Whether the song exists and is publicly visible - what lets another // uploader attach cross-feature content (a custom 3D MV) to it. The // existence check comes first: get_visibility defaults to "public" for an @@ -319,9 +395,32 @@ pub fn all_song_blobs() -> Option { DATABASE.lock_and_select_all("SELECT song FROM songs ORDER BY music_id", params!()).ok() } -// Audio files are content-addressed and may be shared between songs -pub fn audio_in_use(md5: &str, ignored_music_id: i64) -> bool { - DATABASE.lock_and_select("SELECT music_id FROM songs WHERE music_id!=?1 AND song LIKE ?2", params!(ignored_music_id, format!("%{}%", md5))).is_ok() +// Audio files are content-addressed and may be shared between songs, so an ogg is +// only unlinkable once no other song's play/select cue names it. The LIKE scan finds +// candidate rows cheaply and the parsed cues confirm the hit (a substring coincidence +// elsewhere in the blob is not a reference), exactly like custom_3dmv::blob_in_use. +// +// Returns Result and NEVER folds an error into "false": the caller unlinks on false, +// and a read that failed (SQLITE_BUSY under a concurrent write, a corrupt page) says +// nothing about whether the file is referenced. The startup sweep is deliberately +// fail-closed for the same reason; this is the online half of that rule +pub fn audio_in_use(md5: &str, ignored_music_id: i64) -> Result { + let rows = DATABASE.lock_and_select_all( + "SELECT song FROM songs WHERE music_id!=?1 AND song LIKE ?2", + params!(ignored_music_id, format!("%{}%", md5)) + )?; + for blob in rows.members() { + let Ok(song) = jzon::parse(&blob.to_string()) else { + // An unparseable row could reference anything - assume it does + return Ok(true); + }; + for key in ["play", "select"] { + if song["sound"][key]["md5"].as_str() == Some(md5) { + return Ok(true); + } + } + } + Ok(false) } // Resolve a content-addressed chart/jacket md5 to the per-song-id file that @@ -348,3 +447,10 @@ pub fn find_asset_by_md5(md5: &str) -> Option<(i64, String)> { } None } + +// The on-disk database file, so a test can force the read errors the fail-closed +// GC paths are built for +#[cfg(test)] +pub fn test_db_path() -> String { + DATABASE.get_path().to_string() +} diff --git a/src/router/custom_3dmv.rs b/src/router/custom_3dmv.rs index 275f331..21e1d44 100644 --- a/src/router/custom_3dmv.rs +++ b/src/router/custom_3dmv.rs @@ -48,6 +48,19 @@ pub const MAX_FILE_BYTES: usize = 64 * 1024 * 1024; pub const MAX_REQUEST_BYTES: usize = 256 * 1024 * 1024; pub const MAX_MVS_PER_USER: i64 = 200; +// Per-account storage quota, counted over the stored file sizes the catalog +// quotes. The MV count alone bounded one account at 200 x 256MB = ~51GB, which is +// not a bound at all; 4GiB is roughly forty full MVs of realistic size (a model +// zip plus 20-100MB of motion VMDs) and is the largest of the three features' +// quotas because MVs are the heaviest thing a user can upload +pub const MAX_BYTES_PER_USER: i64 = 4 * 1024 * 1024 * 1024; + +// The longest length-prefixed text field a PMX header may declare (model name and +// comment, JP + EN). The prefix is an attacker-supplied i32 the stage walk used to +// skip over by inflating up to 2GB into a sink, four times per entry - CPU with no +// allocation to show for it. Real PMX name/comment fields are tens of bytes +const MAX_PMX_TEXT_BYTES: i32 = 64 * 1024; + // Slots are 1-based, matching the Live3dMemberMst position convention pub const MAX_MEMBER_COUNT: i64 = 12; @@ -206,11 +219,11 @@ async fn read_multipart(mut payload: Multipart) -> Result { while let Some(chunk) = field.try_next().await.map_err(|e| e.to_string())? { total += chunk.len(); if total > MAX_REQUEST_BYTES { - return Err(format!("Upload exceeds the {} MB per-request limit", MAX_REQUEST_BYTES / (1024 * 1024))); + return Err(over_request_limit()); } data.extend_from_slice(&chunk); if data.len() > MAX_FILE_BYTES { - return Err(format!("'{}' exceeds the {} MB per-file limit", name, MAX_FILE_BYTES / (1024 * 1024))); + return Err(over_file_limit(&name)); } } fields.insert(name, data); @@ -218,6 +231,32 @@ async fn read_multipart(mut payload: Multipart) -> Result { Ok(fields) } +pub fn over_file_limit(name: &str) -> String { + format!("'{}' exceeds the {} MB per-file limit", name, MAX_FILE_BYTES / (1024 * 1024)) +} + +pub fn over_request_limit() -> String { + format!("Upload exceeds the {} MB per-request limit", MAX_REQUEST_BYTES / (1024 * 1024)) +} + +// The same accounting read_multipart applies, re-run over a field map that came +// out of an export package. package::expand caps every entry as it inflates, but +// the caps have to hold over the RESULT too: a package is one multipart field and +// its expansion replaces the whole form +fn check_field_caps(fields: &Fields) -> Result<(), String> { + let mut total = 0usize; + for (name, data) in fields.iter() { + if data.len() > MAX_FILE_BYTES { + return Err(over_file_limit(name)); + } + total += data.len(); + if total > MAX_REQUEST_BYTES { + return Err(over_request_limit()); + } + } + Ok(()) +} + fn field_str(fields: &Fields, key: &str) -> String { String::from_utf8_lossy(fields.get(key).map(|v| v.as_slice()).unwrap_or(&[])).trim().to_string() } @@ -263,7 +302,7 @@ fn read_pmx_vertex_count(file: &mut impl Read) -> Option { let mut len = [0u8; 4]; file.read_exact(&mut len).ok()?; let len = i32::from_le_bytes(len); - if len < 0 { + if !(0..=MAX_PMX_TEXT_BYTES).contains(&len) { return None; } if std::io::copy(&mut file.by_ref().take(len as u64), &mut std::io::sink()).ok()? != len as u64 { @@ -436,7 +475,9 @@ fn write_blobs(pending: &[PendingBlob]) -> Result<(), String> { fn gc_blobs(old_files: &JsonValue) { for file in old_files.members() { let md5 = file["md5"].to_string(); - if md5.len() == 32 && !database::blob_in_use(&md5) { + // A read error means "assume referenced": unlinking on a doubtful + // reference set is exactly what the startup sweep refuses to do + if md5.len() == 32 && !database::blob_in_use(&md5).unwrap_or(true) { let _ = fs::remove_file(blob_path(&md5)); } } @@ -477,6 +518,7 @@ pub fn create_mv(uid: i64, fields: &Fields) -> Result { } let (files, pending) = collect_files(fields, member_count, &array![])?; + check_quota(uid, database::mv_bytes(&files), 0)?; let lock = lock_onto_mutex!(UPLOAD_LOCK); let mv_id = database::next_mv_id(); @@ -494,13 +536,27 @@ pub fn create_mv(uid: i64, fields: &Fields) -> Result { }; write_blobs(&pending)?; - database::insert_mv(mv_id, music_id, uid, &mv, published); + database::insert_mv(mv_id, music_id, uid, &mv, published) + .map_err(|e| format!("Could not store the MV: {}", e))?; database::bump_revision(); drop(lock); Ok(mv_id) } +// Per-account storage quota. `excluded` is the MV being replaced by an in-place +// edit - its stored size drops out and `adding` (the resulting size) replaces it +fn check_quota(uid: i64, adding: i64, excluded_mv_id: i64) -> Result<(), String> { + let used = database::owner_bytes(uid, excluded_mv_id); + if used + adding > MAX_BYTES_PER_USER { + return Err(format!( + "This upload would put your MVs at {} MB, over the {} MB per-account limit - delete an MV first", + (used + adding) / (1024 * 1024), MAX_BYTES_PER_USER / (1024 * 1024) + )); + } + Ok(()) +} + // Edit an MV in place. The mv_id and the music_id stay the same: repointing // the song would break the catalog closure for everyone who already resolved // it (delete + re-upload retires the id instead) @@ -523,6 +579,7 @@ pub fn update_mv(uid: i64, mv_id: i64, fields: &Fields) -> Result<(), String> { } let (files, pending) = collect_files(fields, member_count, &stored["files"])?; + check_quota(owner, database::mv_bytes(&files), mv_id)?; let mv = object!{ "mv_id": mv_id, @@ -598,6 +655,30 @@ pub fn purge_song(music_id: i64) { drop(lock); } +// Every MV this account uploaded, gone - called from userdata::delete_account, so +// a purged uploader leaves no catalog row resolving an owner id that no longer +// exists (browse renders an uploader name for every row). Same steps as the +// owner's own delete, blob GC included +pub fn purge_owner(uid: i64) { + if disabled() { + return; + } + let lock = lock_onto_mutex!(UPLOAD_LOCK); + let mut purged = false; + for mv_id in database::mv_ids_for_owner(uid) { + let stored = database::get_mv(mv_id); + database::delete_mv(mv_id); + purged = true; + if let Some(stored) = stored { + gc_blobs(&stored["files"]); + } + } + if purged { + database::bump_revision(); + } + drop(lock); +} + // Startup GC for the content-addressed blob store, mirroring // custom_song::sweep_audio: the only writers are upload and update, so an // unreferenced file is a leftover from an interrupted one. Deliberately @@ -666,6 +747,7 @@ pub fn upload_limits() -> JsonValue { "max_file_bytes": MAX_FILE_BYTES, "max_request_bytes": MAX_REQUEST_BYTES, "max_mvs_per_user": MAX_MVS_PER_USER, + "max_bytes_per_user": MAX_BYTES_PER_USER, "stages": STAGES.to_vec(), "default_stage": STAGES[0], "roles": { @@ -690,21 +772,29 @@ async fn upload(req: HttpRequest, payload: Multipart) -> HttpResponse { Ok(fields) => fields, Err(e) => return webui::error(&e) }; - // An export package from another server: its contents map 1:1 onto the - // normal upload fields, so importing is just an upload - if let Some(bytes) = fields.remove("package") { - if !bytes.is_empty() { - if let Err(e) = package::expand(&bytes, &mut fields) { - return webui::error(&e); + // Zip inflation, the PMX/VMD structure walks, hashing and writing up to 256MB: + // all of it on the blocking pool rather than on the actix worker that also has + // to keep serving the game API + let result = web::block(move || { + // An export package from another server: its contents map 1:1 onto the + // normal upload fields, so importing is just an upload + if let Some(bytes) = fields.remove("package") { + if !bytes.is_empty() { + package::expand(&bytes, &mut fields)?; + // The expansion replaced the form: it has to satisfy the same + // per-file/per-request caps the multipart reader enforces + check_field_caps(&fields)?; } } - } - match create_mv(uid, &fields) { - Ok(mv_id) => send_json(object!{ + create_mv(uid, &fields) + }).await; + match result { + Ok(Ok(mv_id)) => send_json(object!{ result: "OK", mv_id: mv_id }), - Err(e) => webui::error(&e) + Ok(Err(e)) => webui::error(&e), + Err(_) => webui::error("The upload could not be processed") } } @@ -720,12 +810,13 @@ async fn update(req: HttpRequest, payload: Multipart) -> HttpResponse { Err(e) => return webui::error(&e) }; let mv_id = field_str(&fields, "mv_id").parse::().unwrap_or(0); - match update_mv(uid, mv_id, &fields) { - Ok(()) => send_json(object!{ + match web::block(move || update_mv(uid, mv_id, &fields)).await { + Ok(Ok(())) => send_json(object!{ result: "OK", mv_id: mv_id }), - Err(e) => webui::error(&e) + Ok(Err(e)) => webui::error(&e), + Err(_) => webui::error("The edit could not be processed") } } @@ -806,7 +897,15 @@ async fn download(req: HttpRequest) -> HttpResponse { let Some(owner) = database::get_mv_owner(mv_id) else { return webui::error("MV not found"); }; - if get_session_uid(&req) != Some(owner) && !database::is_published(mv_id) { + let viewer = get_session_uid(&req); + if viewer != Some(owner) && !database::is_published(mv_id) { + return webui::error("MV not found"); + } + // Every catalog (list, browse, mine) additionally closes over the songs the + // viewer can see, so a published MV attached to someone else's PRIVATE song is + // invisible everywhere - it must not be downloadable by walking mv_ids either + let music_id = database::get_mv_music_id(mv_id).unwrap_or(0); + if viewer != Some(owner) && !allowed_music_ids(viewer.unwrap_or(0)).contains(&music_id) { return webui::error("MV not found"); } match package::build(mv_id) { @@ -940,7 +1039,7 @@ pub mod tests { "music_id": music_id, "name": format!("Seed Song {}", music_id), "sound": { "play": { "md5": "0".repeat(32) }, "select": { "md5": "0".repeat(32) } } - }, visibility, &array![], false); + }, visibility, &array![], false).unwrap(); } // A complete, valid 2-slot upload: model+motion per slot, a facial on @@ -1420,9 +1519,9 @@ pub mod tests { assert!(!stranger_catalog.members().any(|m| m["mv_id"] == private_song_mv)); // Sharing the song brings its MV along - crate::database::custom_song::set_visibility(970012, "shared", &array![stranger]); + crate::database::custom_song::set_visibility(970012, "shared", &array![stranger]).unwrap(); assert!(catalog_for_user(stranger).members().any(|m| m["mv_id"] == private_song_mv)); - crate::database::custom_song::set_visibility(970012, "private", &array![]); + crate::database::custom_song::set_visibility(970012, "private", &array![]).unwrap(); wipe(owner); } @@ -1501,4 +1600,154 @@ pub mod tests { let _ = fs::remove_file(&junk); wipe(uid); } + + // ---- defect-fix coverage ------------------------------------------------- + + // A stage PMX whose header declares a text field of `len` bytes, with the + // bytes actually present + fn stage_zip_with_text_len(len: i32, present: usize) -> Vec { + let mut pmx = Vec::new(); + pmx.extend(b"PMX "); + pmx.extend(2.0f32.to_le_bytes()); + pmx.push(8); + pmx.extend([0u8; 8]); + // The model name carries the declared length; the other three are empty + pmx.extend(len.to_le_bytes()); + pmx.extend(vec![0u8; present]); + for _ in 0..3 { + pmx.extend(0u32.to_le_bytes()); + } + pmx.extend(1i32.to_le_bytes()); + zip_with("stage.pmx", &pmx) + } + + // D11: the length prefix of a PMX text field is an attacker-supplied i32 the + // stage walk skips over. Uncapped it inflated up to 2GB into a sink, four + // times per entry - CPU with nothing to show for it + #[test] + fn pmx_text_fields_are_bounded() { + let big = MAX_PMX_TEXT_BYTES as usize; + // At the cap, with the bytes really there: a valid stage + assert!(validate_file("stage", "stage", &stage_zip_with_text_len(big as i32, big)).is_ok()); + // One byte over, bytes present: refused rather than skipped over + let err = validate_file("stage", "stage", &stage_zip_with_text_len(big as i32 + 1, big + 1)).unwrap_err(); + assert!(err.contains("malformed PMX header"), "{}", err); + // And the classic: a huge declaration with nothing behind it + let err = validate_file("stage", "stage", &stage_zip_with_text_len(i32::MAX, 0)).unwrap_err(); + assert!(err.contains("malformed PMX header"), "{}", err); + } + + // D1: a package entry is capped as it inflates. Deflate's ~1032:1 ceiling + // means an uncapped read_to_end turns a tiny zip into gigabytes, and a package + // carries 39 addressable entries + #[test] + fn package_import_is_capped() { + let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); + let options = zip::write::SimpleFileOptions::default(); + zip.start_file("manifest.json", options).unwrap(); + zip.write_all(jzon::stringify(object!{ + "format": 1, "name": "Bomb", "name_en": "Bomb", "music_id": 970050, "member_count": 1 + }).as_bytes()).unwrap(); + // Compresses to a few KB, inflates to just over the per-file cap + zip.start_file("model_1", options).unwrap(); + zip.write_all(&vec![0u8; MAX_FILE_BYTES + 1]).unwrap(); + let package = zip.finish().unwrap().into_inner(); + assert!(package.len() < 1024 * 1024, "the bomb should be small: {} bytes", package.len()); + + let mut fields = Fields::new(); + let err = package::expand(&package, &mut fields).unwrap_err(); + assert!(err.contains("per-file limit"), "{}", err); + assert!(fields.get("model_1").is_none(), "the oversized entry was buffered anyway"); + } + + // D1/D6: the per-request total is accounted over the whole form, which is what + // a package's expanded contents are re-checked against + #[test] + fn the_request_total_is_capped() { + let mut fields = Fields::new(); + fields.insert(String::from("a"), vec![0; MAX_FILE_BYTES]); + assert!(check_field_caps(&fields).is_ok()); + let mut one = Fields::new(); + one.insert(String::from("model_1"), vec![0; MAX_FILE_BYTES + 1]); + assert!(check_field_caps(&one).unwrap_err().contains("per-file limit")); + } + + // D2: a read error must never read as "not referenced". The GC unlinks on + // false, so a failed lookup has to mean "assume in use" - the startup sweep + // has always been fail-closed and the online path now matches it + #[test] + fn a_database_error_never_deletes_a_blob() { + let _lock = crate::runtime::lock_test_data_path(); + wipe(9_100_030); + seed_song(970201, 9_100_030, "public"); + let id = create_mv(9_100_030, &base_fields(970201, 1, 120)).unwrap(); + let stored = database::get_mv(id).unwrap(); + let md5 = stored["files"][0]["md5"].to_string(); + assert_eq!(md5.len(), 32); + assert_eq!(database::blob_in_use(&md5), Ok(true)); + assert_eq!(database::blob_in_use(&"c7".repeat(16)), Ok(false)); + + let conn = rusqlite::Connection::open(database::test_db_path()).unwrap(); + conn.execute("ALTER TABLE mvs RENAME TO mvs_hidden", ()).unwrap(); + assert!(database::blob_in_use(&md5).is_err()); + // The blob is still live; the GC must keep what it cannot prove is orphaned + gc_blobs(&stored["files"]); + conn.execute("ALTER TABLE mvs_hidden RENAME TO mvs", ()).unwrap(); + + assert!(fs::read(blob_path(&md5)).is_ok(), "a live blob was unlinked on a read error"); + wipe(9_100_030); + } + + // D9: every catalog closes over the songs the viewer can see, so a published + // MV attached to someone else's PRIVATE song is invisible everywhere - and it + // must not be downloadable by walking mv_ids either + #[test] + fn download_closes_over_song_visibility() { + let _lock = crate::runtime::lock_test_data_path(); + wipe(9_100_031); + seed_song(970202, 9_100_031, "private"); + seed_song(970203, 9_100_031, "public"); + let hidden = create_mv(9_100_031, &base_fields(970202, 1, 130)).unwrap(); + let open = create_mv(9_100_031, &base_fields(970203, 1, 140)).unwrap(); + set_mv_flags(9_100_031, hidden, true).unwrap(); + set_mv_flags(9_100_031, open, true).unwrap(); + + let call = |mv_id: i64| -> String { + let req = actix_web::test::TestRequest::default().param("mv_id", mv_id.to_string()).to_http_request(); + actix_web::rt::System::new().block_on(async { + let resp = download(req).await; + let bytes = actix_web::body::to_bytes(resp.into_body()).await.unwrap(); + String::from_utf8_lossy(&bytes).to_string() + }) + }; + // Published, on a song an anonymous viewer's catalog never delivers + assert!(call(hidden).contains("MV not found"), "a private song's MV was downloadable"); + // Published, on a public song: still downloadable + assert!(!call(open).contains("MV not found")); + + wipe(9_100_031); + } + + // D15: the quota counts the stored file bytes the catalog quotes, per owner + #[test] + fn uploads_are_bounded_by_a_per_account_byte_quota() { + let _lock = crate::runtime::lock_test_data_path(); + wipe(9_100_032); + assert_eq!(database::owner_bytes(9_100_032, 0), 0); + seed_song(970204, 9_100_032, "public"); + let id = create_mv(9_100_032, &base_fields(970204, 1, 150)).unwrap(); + + let stored = database::mv_bytes(&database::get_mv(id).unwrap()["files"]); + assert!(stored > 0); + assert_eq!(database::owner_bytes(9_100_032, 0), stored); + // An in-place edit replaces its own bytes rather than adding to them + assert_eq!(database::owner_bytes(9_100_032, id), 0); + + assert!(check_quota(9_100_032, 1, 0).is_ok()); + assert!(check_quota(9_100_032, MAX_BYTES_PER_USER, 0).unwrap_err().contains("per-account limit")); + assert_eq!(database::owner_bytes(9_100_033, 0), 0); + + wipe(9_100_032); + } + } diff --git a/src/router/custom_3dmv/package.rs b/src/router/custom_3dmv/package.rs index 96a5a1e..60c7c50 100644 --- a/src/router/custom_3dmv/package.rs +++ b/src/router/custom_3dmv/package.rs @@ -42,11 +42,40 @@ pub fn build(mv_id: i64) -> Result, String> { Ok(zip.finish().map_err(|e| e.to_string())?.into_inner()) } -fn read_entry(archive: &mut zip::ZipArchive, name: &str) -> Option> { - let mut file = archive.by_name(name).ok()?; +// Reads one entry, capped. Deflate's ceiling is about 1032:1, so an uncapped +// read_to_end here is a zip bomb: a one-megabyte entry inflates to a gigabyte and +// grows the Vec until the allocator or the OOM killer stops it, and a package has +// 39 addressable entries. Every entry is bounded by the upload form's own +// per-file cap, and by what is left of the per-request budget across all entries. +// +// The central directory's declared size rejects the obvious case without +// inflating anything; take(cap + 1) makes that declaration untrusted - a lying +// header runs out of budget one byte past the cap and stops there. +// +// Ok(None) is "the package does not carry this entry", a normal outcome for every +// optional role +fn read_entry(archive: &mut zip::ZipArchive, name: &str, remaining: &mut usize) -> Result>, String> { + let Ok(mut file) = archive.by_name(name) else { + return Ok(None); + }; + let cap = std::cmp::min(super::MAX_FILE_BYTES, *remaining); + // Which limit the entry actually ran into, so the message names the right one + let too_big = if cap >= super::MAX_FILE_BYTES { + super::over_file_limit(name) + } else { + super::over_request_limit() + }; + if file.size() > cap as u64 { + return Err(too_big); + } let mut bytes = Vec::new(); - file.read_to_end(&mut bytes).ok()?; - Some(bytes) + file.by_ref().take(cap as u64 + 1).read_to_end(&mut bytes) + .map_err(|_| format!("Package entry '{}' could not be read", name))?; + if bytes.len() > cap { + return Err(too_big); + } + *remaining -= bytes.len(); + Ok(Some(bytes)) } // Expands a package into the same field map the upload form produces. The @@ -55,8 +84,10 @@ fn read_entry(archive: &mut zip::ZipArchive, name: &str) -> O // in when the form left it blank (same-server re-upload) pub fn expand(package: &[u8], fields: &mut HashMap>) -> Result<(), String> { let mut archive = zip::ZipArchive::new(Cursor::new(package)).map_err(|_| String::from("Package is not a valid zip file"))?; + // The decompressed budget for the whole package, shared by every entry + let mut remaining = super::MAX_REQUEST_BYTES; - let manifest = read_entry(&mut archive, "manifest.json").ok_or(String::from("Package has no manifest.json"))?; + let manifest = read_entry(&mut archive, "manifest.json", &mut remaining)?.ok_or(String::from("Package has no manifest.json"))?; let manifest = jzon::parse(&String::from_utf8_lossy(&manifest)).map_err(|_| String::from("Package manifest is not valid JSON"))?; if manifest["format"].as_i64() != Some(1) { return Err(String::from("Unsupported package format")); @@ -74,13 +105,13 @@ pub fn expand(package: &[u8], fields: &mut HashMap>) -> Result<( let member_count = manifest["member_count"].as_i64().unwrap_or(0); for slot in 1..=member_count.clamp(0, super::MAX_MEMBER_COUNT) { for role in ["model", "motion", "facial"] { - if let Some(bytes) = read_entry(&mut archive, &format!("{}_{}", role, slot)) { + if let Some(bytes) = read_entry(&mut archive, &format!("{}_{}", role, slot), &mut remaining)? { fields.insert(format!("{}_{}", role, slot), bytes); } } } for name in ["camera", "config", "stage"] { - if let Some(bytes) = read_entry(&mut archive, name) { + if let Some(bytes) = read_entry(&mut archive, name, &mut remaining)? { fields.insert(String::from(name), bytes); } } diff --git a/src/router/custom_card.rs b/src/router/custom_card.rs index 4dd6b4b..8d1b424 100644 --- a/src/router/custom_card.rs +++ b/src/router/custom_card.rs @@ -52,6 +52,13 @@ pub const MAX_FILE_BYTES: usize = 8 * 1024 * 1024; pub const MAX_REQUEST_BYTES: usize = 64 * 1024 * 1024; pub const MAX_CARDS_PER_USER: i64 = 500; +// Per-account storage quota over the stored art and voiceline bytes, shared by +// cards and characters (they share one storage root). One card derives 14 PNGs up +// to 2048x1260, so the 500-card count limit alone allowed several gigabytes per +// account with no byte bound at all; 2GiB is around a hundred full cards, well +// past any real uploader, and matches the custom-song quota +pub const MAX_BYTES_PER_USER: i64 = 2 * 1024 * 1024 * 1024; + // Columns the uploader never supplies. master_release_label_id must be 1: a // closed label filters the card out of the member-list filters and drops its // evolve conditions @@ -127,6 +134,24 @@ const CHARACTER_ART: &[ArtKind] = &[ ArtKind { kind: "character", width: 600, height: 920 } ]; +// One uploadable voiceline moment. `name` is the wire kind: it is the infix of +// the multipart field names below, the "kind" of every stored/served line, and +// what the client maps to a SYSTEM_VOICE_TYPE (and, for the day_* moments, to +// the date_condition id the game's resolver compares). `category` only groups +// the upload form. +// +// The home-screen moments (category "home") are exactly the SYSTEM_VOICE_TYPEs +// HomeScene.CreateVoiceData asks the resolver for; everything else is a moment +// that already shipped. Kind names are the snake_case of the game's own +// SYSTEM_VOICE_TYPE members (day_* follow the game's own SYS_VOICE_DAY_MMDD cue +// naming), so the mapping stays readable on both sides. +pub struct VoiceKind { + pub name: &'static str, + pub label: &'static str, + pub label_en: &'static str, + pub category: &'static str, +} + // Optional voicelines per character. Multipart fields per line: // voice_{kind}_{index} audio file (any format symphonia reads; // transcoded to ogg-vorbis, stored @@ -137,9 +162,71 @@ const CHARACTER_ART: &[ArtKind] = &[ // Absent slots keep their stored line, captions without a file update the // stored line's captions, and surviving lines are renumbered contiguously // per kind (1..n) after every edit -const VOICE_KINDS: &[&str] = &[ - "live_start", "live_success", "live_failed", "result_bond", - "skill_smile", "skill_pure", "skill_cool" +pub const VOICE_KINDS: &[VoiceKind] = &[ + // --- moments outside the home screen (unchanged wire names) --- + VoiceKind { name: "live_start", label: "ライブ開始（リーダー）", label_en: "Live start (leader, before a live)", category: "live" }, + VoiceKind { name: "live_success", label: "ライブクリア", label_en: "Live cleared", category: "live" }, + VoiceKind { name: "live_failed", label: "ライブ失敗", label_en: "Live failed", category: "live" }, + VoiceKind { name: "result_bond", label: "リザルト・絆獲得", label_en: "Bond gained (live result)", category: "result" }, + VoiceKind { name: "skill_smile", label: "スキル発動（スマイル）", label_en: "Skill activation - Smile cards", category: "skill" }, + VoiceKind { name: "skill_pure", label: "スキル発動（ピュア）", label_en: "Skill activation - Pure cards", category: "skill" }, + VoiceKind { name: "skill_cool", label: "スキル発動（クール）", label_en: "Skill activation - Cool cards", category: "skill" }, + + // --- home screen: always in the pool --- + VoiceKind { name: "random", label: "ランダム", label_en: "Random line (always in the pool)", category: "home" }, + VoiceKind { name: "random_evolution", label: "ランダム（覚醒後）", label_en: "Random line, awakened card only", category: "home" }, + VoiceKind { name: "random_unevolution_smile", label: "ランダム（未覚醒・スマイル）", label_en: "Random line, un-awakened Smile card", category: "home" }, + VoiceKind { name: "random_unevolution_pure", label: "ランダム（未覚醒・ピュア）", label_en: "Random line, un-awakened Pure card", category: "home" }, + VoiceKind { name: "random_unevolution_cool", label: "ランダム（未覚醒・クール）", label_en: "Random line, un-awakened Cool card", category: "home" }, + VoiceKind { name: "random_evolution_smile", label: "ランダム（覚醒後・スマイル）", label_en: "Random line, awakened Smile card", category: "home" }, + VoiceKind { name: "random_evolution_pure", label: "ランダム（覚醒後・ピュア）", label_en: "Random line, awakened Pure card", category: "home" }, + VoiceKind { name: "random_evolution_cool", label: "ランダム（覚醒後・クール）", label_en: "Random line, awakened Cool card", category: "home" }, + VoiceKind { name: "touch", label: "タップ", label_en: "Tapped (the touch-to-play-voice button)", category: "home" }, + + // --- home screen: time of day (the game's own hour bands) --- + VoiceKind { name: "time", label: "時間帯（共通）", label_en: "Any time of day", category: "home" }, + VoiceKind { name: "time_firsthalf", label: "時間帯（5時〜17時）", label_en: "First half of the day (05-17)", category: "home" }, + VoiceKind { name: "time_latterhalf", label: "時間帯（17時〜5時）", label_en: "Latter half of the day (17-05)", category: "home" }, + VoiceKind { name: "time_morning", label: "時間帯（朝・5時〜11時）", label_en: "Morning (05-11)", category: "home" }, + VoiceKind { name: "time_noon", label: "時間帯（昼・11時〜17時）", label_en: "Daytime (11-17)", category: "home" }, + VoiceKind { name: "time_evening", label: "時間帯（夕方・17時〜23時）", label_en: "Evening (17-23)", category: "home" }, + VoiceKind { name: "time_night", label: "時間帯（夜・23時〜5時）", label_en: "Night (23-05)", category: "home" }, + + // --- home screen: season (the game's own month bands) --- + VoiceKind { name: "season_spring", label: "季節（春・3〜5月）", label_en: "Spring (March-May)", category: "home" }, + VoiceKind { name: "season_summer", label: "季節（夏・6〜8月）", label_en: "Summer (June-August)", category: "home" }, + VoiceKind { name: "season_autumn", label: "季節（秋・9〜11月）", label_en: "Autumn (September-November)", category: "home" }, + VoiceKind { name: "season_winter", label: "季節（冬・12〜2月）", label_en: "Winter (December-February)", category: "home" }, + + // --- home screen: calendar days. On a day that matches, the home screen + // asks for NOTHING ELSE (a matching day line pre-empts the whole pool + // unless the player taps), so these are the "special occasion" lines + VoiceKind { name: "day_0101", label: "記念日（1月1日・お正月）", label_en: "January 1 (New Year's Day)", category: "home" }, + VoiceKind { name: "day_0203", label: "記念日（2月3日・節分）", label_en: "February 3 (Setsubun)", category: "home" }, + VoiceKind { name: "day_0214", label: "記念日（2月14日・バレンタイン）", label_en: "February 14 (Valentine's Day)", category: "home" }, + VoiceKind { name: "day_0303", label: "記念日（3月3日・ひな祭り）", label_en: "March 3 (Hinamatsuri)", category: "home" }, + VoiceKind { name: "day_0314", label: "記念日（3月14日・ホワイトデー）", label_en: "March 14 (White Day)", category: "home" }, + VoiceKind { name: "day_0505", label: "記念日（5月5日・こどもの日）", label_en: "May 5 (Children's Day)", category: "home" }, + VoiceKind { name: "day_0707", label: "記念日（7月7日・七夕）", label_en: "July 7 (Tanabata)", category: "home" }, + VoiceKind { name: "day_0717", label: "記念日（7月第3月曜・海の日）", label_en: "Marine Day (third Monday of July)", category: "home" }, + VoiceKind { name: "day_1015", label: "記念日（10月15日）", label_en: "October 15", category: "home" }, + VoiceKind { name: "day_1031", label: "記念日（10月31日・ハロウィン）", label_en: "October 31 (Halloween)", category: "home" }, + VoiceKind { name: "day_1225", label: "記念日（12月25日・クリスマス）", label_en: "December 25 (Christmas)", category: "home" }, + VoiceKind { name: "day_1231", label: "記念日（12月31日・大晦日）", label_en: "December 31 (New Year's Eve)", category: "home" }, + + // --- home screen: lines keyed to what the account has waiting --- + VoiceKind { name: "advice_story", label: "未読ストーリーあり（おすすめ）", label_en: "Unread story waiting (advice line)", category: "home" }, + VoiceKind { name: "trigger_story", label: "未読ストーリーあり（反応）", label_en: "Unread story waiting (trigger line)", category: "home" }, + VoiceKind { name: "advice_mission", label: "デイリーミッション報酬未受取", label_en: "Unclaimed daily mission reward", category: "home" }, + VoiceKind { name: "advice_live", label: "LPが足りている（ライブ）", label_en: "Enough LP to play a live", category: "home" }, + VoiceKind { name: "advice_lp", label: "LP半分以下（回復のすすめ）", label_en: "LP below half, recovery affordable", category: "home" }, + VoiceKind { name: "trigger_shop", label: "LP半分以下（ショップ）", label_en: "LP below half, recovery affordable (shop line)", category: "home" }, + VoiceKind { name: "advice_lesson", label: "レッスン（未カンスト）", label_en: "A unit is not fully levelled", category: "home" }, + VoiceKind { name: "advice_present", label: "プレゼント未受取", label_en: "Unclaimed present waiting", category: "home" }, + VoiceKind { name: "advice_gacha", label: "無料スカウト可能", label_en: "The daily free scouting is available", category: "home" }, + VoiceKind { name: "advice_infomation", label: "新着お知らせあり", label_en: "A new announcement is up", category: "home" }, + VoiceKind { name: "advice_event", label: "イベント開催中", label_en: "An event is running", category: "home" }, + VoiceKind { name: "trigger_friend", label: "イベント中・フレンド", label_en: "An event is running and a friend is active", category: "home" } ]; const MAX_VOICE_VARIANTS: usize = 9; const MAX_VOICE_BYTES: usize = 4 * 1024 * 1024; @@ -482,6 +569,17 @@ pub fn upload_limits() -> JsonValue { }; } let ((prob_min, prob_max), (ms_min, ms_max)) = *SKILL_SCALAR_RANGES; + // The voiceline moments, so the form renders exactly the kinds this build + // accepts instead of carrying its own copy of the list + let mut voice_kinds = array![]; + for kind in VOICE_KINDS { + voice_kinds.push(object!{ + "name": kind.name, + "label": kind.label, + "label_en": kind.label_en, + "category": kind.category + }).unwrap(); + } object!{ "stat_caps": stat_caps, "skill_levels": skill_levels, @@ -498,7 +596,15 @@ pub fn upload_limits() -> JsonValue { "probability": { "min": prob_min, "max": prob_max }, "milli_secs": { "min": ms_min, "max": ms_max } }, - "min_source_dim": art::MIN_SOURCE_DIM + "min_source_dim": art::MIN_SOURCE_DIM, + "max_file_bytes": MAX_FILE_BYTES, + "max_request_bytes": MAX_REQUEST_BYTES, + "max_cards_per_user": MAX_CARDS_PER_USER, + "max_bytes_per_user": MAX_BYTES_PER_USER, + "voice_kinds": voice_kinds, + "max_voice_variants": MAX_VOICE_VARIANTS, + "max_voice_bytes": MAX_VOICE_BYTES, + "max_voice_seconds": MAX_VOICE_SECONDS } } @@ -559,6 +665,17 @@ async fn data(req: HttpRequest) -> HttpResponse { return HttpResponse::NotFound().finish(); }; match fs::read(asset_path(&relative)) { + // Art lives at FIXED per-card filenames ({kind}_{variant}.png) that an + // in-place edit overwrites, so between the file write and the catalog + // update the index still points an old md5 at bytes that are no longer its + // own. The client caches whatever it downloads under the md5 it asked for + // and never re-checks, so serving those bytes would poison its cache + // permanently. The index is a hint; these bytes are the answer only if + // they hash to the request. A mismatch is the same 404 a stale md5 already + // gets, and the client re-downloads under the md5 the catalog now carries + Ok(body) if !hash.eq_ignore_ascii_case(&format!("{:x}", md5::compute(&body))) => { + HttpResponse::NotFound().finish() + }, Ok(body) => { HttpResponse::Ok() .insert_header(ContentType::png()) @@ -840,11 +957,29 @@ fn write_art(dir: &str, pending: &[PendingArt]) -> Result<(), String> { fn collect_voice(fields: &Fields, stored_voice: &JsonValue) -> Result<(JsonValue, Vec<(String, Vec)>), String> { let mut rv = array![]; let mut files: Vec<(String, Vec)> = Vec::new(); + // An unknown moment is a typo, and silently ignoring it (what happens to + // every other unrecognised multipart field) would show the uploader a clip + // that uploaded fine and a line that never appeared. An out-of-range INDEX + // stays ignored - that is a full slot list, not a misspelling + for key in fields.keys() { + let Some(rest) = key.strip_prefix("voice_") else { continue; }; + let rest = ["_text_en", "_text", "_delete"].iter() + .find_map(|suffix| rest.strip_suffix(suffix)) + .unwrap_or(rest); + let named_moment = match rest.rsplit_once('_') { + Some((kind, index)) => index.parse::().is_ok() && VOICE_KINDS.iter().any(|k| k.name == kind), + None => false + }; + if !named_moment { + return Err(format!("'{}' is not a voiceline field", key)); + } + } for kind in VOICE_KINDS { + let kind = kind.name; let mut lines: Vec = Vec::new(); for index in 1..=MAX_VOICE_VARIANTS { let base = format!("voice_{}_{}", kind, index); - let stored_line = stored_voice.members().find(|line| line["kind"] == *kind && line["index"] == index); + let stored_line = stored_voice.members().find(|line| line["kind"] == kind && line["index"] == index); if field_flag(fields, &format!("{}_delete", base)) { if file_of(fields, &base).is_some() { return Err(format!("'{}': cannot both replace and delete the same line", base)); @@ -869,7 +1004,7 @@ fn collect_voice(fields: &Fields, stored_voice: &JsonValue) -> Result<(JsonValue } let clip = audio::process_one_shot(bytes, MAX_VOICE_SECONDS).map_err(|e| format!("'{}': {}", base, e))?; lines.push(object!{ - "kind": *kind, + "kind": kind, "index": 0, "md5": clip.md5.clone(), "size": clip.bytes.len(), @@ -879,7 +1014,7 @@ fn collect_voice(fields: &Fields, stored_voice: &JsonValue) -> Result<(JsonValue files.push((clip.md5, clip.bytes)); } else if let Some(stored_line) = stored_line { lines.push(object!{ - "kind": *kind, + "kind": kind, "index": 0, "md5": stored_line["md5"].clone(), "size": stored_line["size"].clone(), @@ -917,9 +1052,12 @@ fn write_voice(master_character_id: i64, files: &[(String, Vec)]) -> Result< // (their old catalog md5 404s, exactly like replaced art) fn gc_voice(master_character_id: i64, old_voice: &JsonValue, new_voice: &JsonValue) { for old in old_voice.members() { - let md5 = old["md5"].to_string(); - if !md5.is_empty() && !new_voice.members().any(|line| line["md5"] == old["md5"]) { - let _ = fs::remove_file(voice_path(master_character_id, &md5)); + // JsonValue::to_string renders Null as the literal "null", which an + // is_empty() guard happily passes; only exactly 32 hex characters is a + // hash (the same test custom_3dmv's blob GC uses) + let Some(md5) = old["md5"].as_str().filter(|md5| md5.len() == 32 && md5.chars().all(|c| c.is_ascii_hexdigit())) else { continue; }; + if !new_voice.members().any(|line| line["md5"] == old["md5"]) { + let _ = fs::remove_file(voice_path(master_character_id, md5)); } } } @@ -1215,15 +1353,31 @@ pub fn create_card(uid: i64, fields: &Fields) -> Result { let mut card = build_card(master_card_id, master_character_id, fields, &object!{})?; card["art"] = merge_art(&array![], &card_art); + check_quota(uid, database::card_bytes(&card), 0, 0)?; write_art(&card_dir(master_card_id), &card_art)?; - database::insert_card(master_card_id, master_character_id, uid, &card, published, obtainable); + database::insert_card(master_card_id, master_character_id, uid, &card, published, obtainable) + .map_err(|e| format!("Could not store the card: {}", e))?; database::bump_revision(); drop(lock); Ok(master_card_id) } +// Per-account storage quota over this account's cards AND characters. The +// excluded ids are the entity being replaced by an in-place edit - its stored +// size drops out and `adding` (the resulting size) replaces it +fn check_quota(uid: i64, adding: i64, excluded_card_id: i64, excluded_character_id: i64) -> Result<(), String> { + let used = database::owner_bytes(uid, excluded_card_id, excluded_character_id); + if used + adding > MAX_BYTES_PER_USER { + return Err(format!( + "This upload would put your uploads at {} MB, over the {} MB per-account limit - delete a card or character first", + (used + adding) / (1024 * 1024), MAX_BYTES_PER_USER / (1024 * 1024) + )); + } + Ok(()) +} + // Edit a card in place. The master_card_id - and everything derived from it: // master_skill_id, illust ids - stays the same, so a player who owns the card // keeps owning the same card. master_character_id is fixed too: repointing it @@ -1242,6 +1396,7 @@ pub fn update_card(uid: i64, master_card_id: i64, fields: &Fields) -> Result<(), let mut card = build_card(master_card_id, master_character_id, fields, &stored)?; let card_art = collect_card_art(fields, false)?; card["art"] = merge_art(&stored["art"], &card_art); + check_quota(owner, database::card_bytes(&card), master_card_id, 0)?; let lock = lock_onto_mutex!(UPLOAD_LOCK); write_art(&card_dir(master_card_id), &card_art)?; @@ -1313,10 +1468,12 @@ pub fn create_character(uid: i64, fields: &Fields) -> Result { if !voice.is_empty() { character["voice"] = voice; } + check_quota(uid, database::character_bytes(&character), 0, 0)?; write_art(&character_dir(master_character_id), &character_art)?; write_voice(master_character_id, &voice_files)?; - database::insert_character(master_character_id, uid, &character); + database::insert_character(master_character_id, uid, &character) + .map_err(|e| format!("Could not store the character: {}", e))?; database::bump_revision(); drop(lock); @@ -1340,6 +1497,7 @@ pub fn update_character(uid: i64, master_character_id: i64, fields: &Fields) -> if !voice.is_empty() { character["voice"] = voice.clone(); } + check_quota(owner, database::character_bytes(&character), 0, master_character_id)?; let lock = lock_onto_mutex!(UPLOAD_LOCK); write_art(&character_dir(master_character_id), &character_art)?; @@ -1376,6 +1534,41 @@ pub fn delete_character(uid: i64, master_character_id: i64) -> Result<(), String Ok(()) } +// Every card this account uploaded, gone - called from userdata::delete_account, +// so a purged uploader leaves no catalog row resolving an owner id that no longer +// exists (browse renders an uploader name for every row). Player copies of the +// dead cards are wiped lazily on each account's next userdata pull, exactly like +// an owner-initiated delete. +// +// Characters go with them only when nothing else references them: a custom +// character that still backs SOMEONE ELSE'S card cannot be deleted without +// serving a dangling master_character_id, which the client throws on. Those stay, +// ownerless, which is the same trade delete_character already makes +pub fn purge_owner(uid: i64) { + if disabled() { + return; + } + let cards = database::card_ids_for_owner(uid); + let characters = database::character_ids_for_owner(uid); + if cards.is_empty() && characters.is_empty() { + return; + } + let lock = lock_onto_mutex!(UPLOAD_LOCK); + for master_card_id in cards { + database::delete_card(master_card_id); + let _ = fs::remove_dir_all(card_dir(master_card_id)); + } + for master_character_id in characters { + if database::cards_using_character(master_character_id) > 0 { + continue; + } + database::delete_character(master_character_id); + let _ = fs::remove_dir_all(character_dir(master_character_id)); + } + database::bump_revision(); + drop(lock); +} + async fn create(req: HttpRequest, payload: Multipart) -> HttpResponse { if disabled() { return HttpResponse::NotFound().finish(); @@ -1387,12 +1580,16 @@ async fn create(req: HttpRequest, payload: Multipart) -> HttpResponse { Ok(fields) => fields, Err(e) => return webui::error(&e) }; - match create_card(uid, &fields) { - Ok(master_card_id) => send_json(object!{ + // Deriving 14 Lanczos3 PNGs up to 2048x1260 (and transcoding voicelines) is + // seconds of CPU: it belongs on the blocking pool, not on the actix worker + // that also has to keep serving the game API + match web::block(move || create_card(uid, &fields)).await { + Ok(Ok(master_card_id)) => send_json(object!{ result: "OK", master_card_id: master_card_id }), - Err(e) => webui::error(&e) + Ok(Err(e)) => webui::error(&e), + Err(_) => webui::error("The upload could not be processed") } } @@ -1408,12 +1605,16 @@ async fn update(req: HttpRequest, payload: Multipart) -> HttpResponse { Err(e) => return webui::error(&e) }; let master_card_id = field_str(&fields, "master_card_id").parse::().unwrap_or(0); - match update_card(uid, master_card_id, &fields) { - Ok(()) => send_json(object!{ + // Deriving 14 Lanczos3 PNGs up to 2048x1260 (and transcoding voicelines) is + // seconds of CPU: it belongs on the blocking pool, not on the actix worker + // that also has to keep serving the game API + match web::block(move || update_card(uid, master_card_id, &fields)).await { + Ok(Ok(())) => send_json(object!{ result: "OK", master_card_id: master_card_id }), - Err(e) => webui::error(&e) + Ok(Err(e)) => webui::error(&e), + Err(_) => webui::error("The edit could not be processed") } } @@ -1461,12 +1662,16 @@ async fn character_create(req: HttpRequest, payload: Multipart) -> HttpResponse Ok(fields) => fields, Err(e) => return webui::error(&e) }; - match create_character(uid, &fields) { - Ok(master_character_id) => send_json(object!{ + // Deriving 14 Lanczos3 PNGs up to 2048x1260 (and transcoding voicelines) is + // seconds of CPU: it belongs on the blocking pool, not on the actix worker + // that also has to keep serving the game API + match web::block(move || create_character(uid, &fields)).await { + Ok(Ok(master_character_id)) => send_json(object!{ result: "OK", master_character_id: master_character_id }), - Err(e) => webui::error(&e) + Ok(Err(e)) => webui::error(&e), + Err(_) => webui::error("The upload could not be processed") } } @@ -1482,12 +1687,16 @@ async fn character_update(req: HttpRequest, payload: Multipart) -> HttpResponse Err(e) => return webui::error(&e) }; let master_character_id = field_str(&fields, "master_character_id").parse::().unwrap_or(0); - match update_character(uid, master_character_id, &fields) { - Ok(()) => send_json(object!{ + // Deriving 14 Lanczos3 PNGs up to 2048x1260 (and transcoding voicelines) is + // seconds of CPU: it belongs on the blocking pool, not on the actix worker + // that also has to keep serving the game API + match web::block(move || update_character(uid, master_character_id, &fields)).await { + Ok(Ok(())) => send_json(object!{ result: "OK", master_character_id: master_character_id }), - Err(e) => webui::error(&e) + Ok(Err(e)) => webui::error(&e), + Err(_) => webui::error("The edit could not be processed") } } @@ -1813,7 +2022,7 @@ pub mod tests { let mut edit = Fields::new(); edit.insert(String::from("voice_result_bond_1"), test_wav(31.0, 6)); let err = with_permissions(4010, &[permissions::CARD_UPLOAD], || update_character(4010, id, &edit)).unwrap_err(); - assert!(err.contains("voice_result_bond_1") && err.contains("maximum is 30"), "{}", err); + assert!(err.contains("voice_result_bond_1") && err.contains("30 second maximum"), "{}", err); let mut edit = Fields::new(); edit.insert(String::from("voice_result_bond_1"), b"definitely not audio".to_vec()); assert!(with_permissions(4010, &[permissions::CARD_UPLOAD], || update_character(4010, id, &edit)) @@ -1823,6 +2032,145 @@ pub mod tests { wipe(4010); } + // The home-screen moments ride the exact same upload path as the live/skill + // ones: real create + edit, transcode, content addressing, renumbering. + // day_1225 is the interesting one - the client turns it into a row carrying + // a date_condition id rather than 0 + #[test] + fn home_voicelines_upload_through_the_real_path() { + let _lock = crate::runtime::lock_test_data_path(); + wipe(4011); + + let mut fields = character_fields(); + fields.insert(String::from("voice_random_1"), test_wav(1.0, 11)); + field(&mut fields, "voice_random_1_text", "おはよう！"); + field(&mut fields, "voice_random_1_text_en", "Morning!"); + fields.insert(String::from("voice_random_3"), test_wav(1.0, 12)); + fields.insert(String::from("voice_touch_1"), test_wav(0.5, 13)); + fields.insert(String::from("voice_time_morning_1"), test_wav(0.5, 14)); + fields.insert(String::from("voice_day_1225_1"), test_wav(0.5, 15)); + fields.insert(String::from("voice_trigger_friend_1"), test_wav(0.5, 16)); + // Still ignored: a real moment, an index past the variant cap + fields.insert(String::from("voice_random_10"), test_wav(1.0, 17)); + let id = with_permissions(4011, &[permissions::CARD_UPLOAD], || create_character(4011, &fields).unwrap()); + + let character = database::get_character(id).unwrap(); + let voice = &character["voice"]; + assert_eq!(voice.len(), 6); + let kinds: Vec = voice.members().map(|line| line["kind"].to_string()).collect(); + for kind in ["random", "touch", "time_morning", "day_1225", "trigger_friend"] { + assert!(kinds.contains(&String::from(kind)), "{} missing from {:?}", kind, kinds); + } + // The sparse random indexes renumbered to 1 and 2, captions intact + let random: Vec<&JsonValue> = voice.members().filter(|line| line["kind"] == "random").collect(); + assert_eq!(random.len(), 2); + assert_eq!(random[0]["index"].as_i64(), Some(1)); + assert_eq!(random[1]["index"].as_i64(), Some(2)); + assert_eq!(random[0]["text"].as_str(), Some("おはよう！")); + assert_eq!(random[0]["text_en"].as_str(), Some("Morning!")); + for line in voice.members() { + let md5 = line["md5"].to_string(); + let bytes = fs::read(voice_path(id, &md5)).unwrap(); + assert!(bytes.starts_with(b"OggS"), "transcoded to ogg"); + assert_eq!(database::find_voice_by_md5(&md5), Some(format!("characters/{}/voice/{}.ogg", id, md5))); + } + + // Editing one home moment leaves the others alone + let touch_md5 = voice.members().find(|line| line["kind"] == "touch").unwrap()["md5"].to_string(); + let mut edit = Fields::new(); + field(&mut edit, "voice_day_1225_1_text", "メリークリスマス！"); + edit.insert(String::from("voice_touch_2"), test_wav(0.5, 18)); + with_permissions(4011, &[permissions::CARD_UPLOAD], || update_character(4011, id, &edit).unwrap()); + let after = database::get_character(id).unwrap(); + assert_eq!(after["voice"].len(), 7); + assert_eq!(after["voice"].members().find(|line| line["kind"] == "day_1225").unwrap()["text"].as_str(), + Some("メリークリスマス！")); + assert_eq!(after["voice"].members() + .find(|line| line["kind"] == "touch" && line["index"] == 1).unwrap()["md5"].to_string(), touch_md5); + + // The 30-second and rich-text rules apply to the new moments too + let mut edit = Fields::new(); + edit.insert(String::from("voice_time_night_1"), test_wav(31.0, 19)); + let err = with_permissions(4011, &[permissions::CARD_UPLOAD], || update_character(4011, id, &edit)).unwrap_err(); + assert!(err.contains("voice_time_night_1") && err.contains("30 second maximum"), "{}", err); + let mut edit = Fields::new(); + field(&mut edit, "voice_touch_1_text", "x"); + assert!(with_permissions(4011, &[permissions::CARD_UPLOAD], || update_character(4011, id, &edit)) + .unwrap_err().contains("")); + + wipe(4011); + } + + // A misspelled moment must not upload as silence-that-never-plays + #[test] + fn an_unknown_voice_kind_is_rejected() { + let _lock = crate::runtime::lock_test_data_path(); + wipe(4012); + + let reject = |key: &str| { + let mut fields = character_fields(); + fields.insert(String::from(key), test_wav(0.5, 21)); + let err = with_permissions(4012, &[permissions::CARD_UPLOAD], || create_character(4012, &fields)).unwrap_err(); + assert!(err.contains(key) && err.contains("not a voiceline field"), "{}: {}", key, err); + }; + reject("voice_time_mornng_1"); + reject("voice_day_0102_1"); + reject("voice_live_start"); + reject("voice_random_x"); + + // Captions and delete flags go through the same check + let mut fields = character_fields(); + field(&mut fields, "voice_seaon_spring_1_text", "x"); + let err = with_permissions(4012, &[permissions::CARD_UPLOAD], || create_character(4012, &fields)).unwrap_err(); + assert!(err.contains("voice_seaon_spring_1_text"), "{}", err); + + // ...and a correctly named one still goes through + let mut fields = character_fields(); + fields.insert(String::from("voice_season_spring_1"), test_wav(0.5, 22)); + let id = with_permissions(4012, &[permissions::CARD_UPLOAD], || create_character(4012, &fields).unwrap()); + assert_eq!(database::get_character(id).unwrap()["voice"][0]["kind"].as_str(), Some("season_spring")); + + wipe(4012); + } + + // The webui renders whatever this serves; a kind that only exists in one of + // the two lists is the bug this endpoint exists to prevent + #[test] + fn the_limits_endpoint_lists_the_voice_kinds() { + let limits = upload_limits(); + assert_eq!(limits["max_voice_variants"].as_usize(), Some(MAX_VOICE_VARIANTS)); + assert_eq!(limits["max_voice_bytes"].as_usize(), Some(MAX_VOICE_BYTES)); + assert_eq!(limits["max_voice_seconds"].as_f64(), Some(MAX_VOICE_SECONDS)); + let served = &limits["voice_kinds"]; + assert_eq!(served.len(), VOICE_KINDS.len()); + for (entry, kind) in served.members().zip(VOICE_KINDS) { + assert_eq!(entry["name"].as_str(), Some(kind.name)); + assert_eq!(entry["label"].as_str(), Some(kind.label)); + assert_eq!(entry["label_en"].as_str(), Some(kind.label_en)); + assert_eq!(entry["category"].as_str(), Some(kind.category)); + assert!(!kind.label.is_empty() && !kind.label_en.is_empty(), "{} needs both labels", kind.name); + assert!(["home", "live", "result", "skill"].contains(&kind.category), "{}: {}", kind.name, kind.category); + } + // Names are the multipart infix and the client's switch arm: unique, + // snake_case, and free of the separators the field parser splits on + let mut names: Vec<&str> = VOICE_KINDS.iter().map(|kind| kind.name).collect(); + names.sort_unstable(); + let count = names.len(); + names.dedup(); + assert_eq!(names.len(), count, "duplicate voice kind name"); + for kind in VOICE_KINDS { + assert!(kind.name.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_'), "{}", kind.name); + // The field parser strips these suffixes before splitting off the index + assert!(!kind.name.ends_with("_text") && !kind.name.ends_with("_text_en") && !kind.name.ends_with("_delete"), "{}", kind.name); + // ...and splits at the LAST underscore, so "{another kind}_{number}" + // would be two readings of the same field name + if let Some((head, tail)) = kind.name.rsplit_once('_') { + assert!(tail.parse::().is_err() || !VOICE_KINDS.iter().any(|other| other.name == head), + "{} is ambiguous with {}", kind.name, head); + } + } + } + // Diagnostic + sanity guard for the derived skill magnitude envelopes #[test] fn skill_ranges_derive_from_shipped_rows() { @@ -2274,4 +2622,106 @@ pub mod tests { assert!(!viewer_can_resolve(100_010_001, 1)); assert!(viewer_can_resolve(100_010_001, 2)); } + + // ---- defect-fix coverage ------------------------------------------------- + + // D14: card art lives at fixed per-card filenames that an in-place edit + // overwrites, so the md5 index can briefly point at bytes that are no longer + // its own. The client caches by md5 and never re-checks, so the route verifies + // the bytes before it serves them + #[test] + fn the_data_route_refuses_bytes_that_do_not_match_the_md5() { + let _lock = crate::runtime::lock_test_data_path(); + wipe(4090); + + let png = seeded_png(120, 60, 91); + let md5 = format!("{:x}", md5::compute(&png)); + let id = database::next_card_id(); + database::insert_card(id, 1001, 4090, &object!{ + "master_card_id": id, + "rarity": 1, + "art": [{ "kind": "c", "variant": "00", "md5": md5.clone(), "size": png.len() }] + }, true, true).unwrap(); + fs::create_dir_all(card_dir(id)).unwrap(); + let file = format!("{}/c_00.png", card_dir(id)); + fs::write(&file, &png).unwrap(); + + let call = || -> actix_web::http::StatusCode { + let req = actix_web::test::TestRequest::default() + .param("hash", md5.clone()) + .param("file", format!("{}.png", md5)) + .to_http_request(); + actix_web::rt::System::new().block_on(async { data(req).await }).status() + }; + assert_eq!(call(), actix_web::http::StatusCode::OK); + + // The index still resolves, but the file no longer holds those bytes + fs::write(&file, seeded_png(120, 60, 92)).unwrap(); + assert_eq!(call(), actix_web::http::StatusCode::NOT_FOUND); + fs::write(&file, &png).unwrap(); + assert_eq!(call(), actix_web::http::StatusCode::OK); + + wipe(4090); + } + + // D13: JsonValue::to_string renders Null as the literal "null", so a stored + // line with no md5 used to aim the unlink at a file called null.ogg + #[test] + fn the_voice_gc_ignores_a_missing_md5() { + let _lock = crate::runtime::lock_test_data_path(); + let character = 5_900_001; + let dir = format!("{}/voice", character_dir(character)); + fs::create_dir_all(&dir).unwrap(); + let decoy = format!("{}/null.ogg", dir); + fs::write(&decoy, b"not a voiceline").unwrap(); + let real_md5 = "b1".repeat(16); + let real = voice_path(character, &real_md5); + fs::write(&real, b"a voiceline").unwrap(); + + let old = array![ + { "kind": "live_start", "index": 1, "md5": JsonValue::Null }, + { "kind": "live_start", "index": 2, "md5": real_md5.clone() } + ]; + gc_voice(character, &old, &array![]); + + assert!(fs::read(&decoy).is_ok(), "the GC unlinked a file named after JSON null"); + assert!(fs::read(&real).is_err(), "the dropped line's ogg was kept"); + let _ = fs::remove_dir_all(character_dir(character)); + } + + // D15: cards and characters share one storage root, so they share one + // per-account byte quota + #[test] + fn uploads_are_bounded_by_a_per_account_byte_quota() { + let _lock = crate::runtime::lock_test_data_path(); + wipe(4091); + assert_eq!(database::owner_bytes(4091, 0, 0), 0); + + let id = database::next_card_id(); + database::insert_card(id, 1001, 4091, &object!{ + "master_card_id": id, + "rarity": 1, + "art": [{ "kind": "c", "variant": "00", "md5": "a1".repeat(16), "size": 1000 }] + }, true, true).unwrap(); + let character = database::next_character_id(); + database::insert_character(character, 4091, &object!{ + "master_character_id": character, + "name": "Quota", + "art": [{ "kind": "icon", "md5": "a2".repeat(16), "size": 500 }], + "voice": [{ "kind": "live_start", "index": 1, "md5": "a3".repeat(16), "size": 250 }] + }).unwrap(); + + assert_eq!(database::owner_bytes(4091, 0, 0), 1750); + // An in-place edit replaces its own bytes rather than adding to them + assert_eq!(database::owner_bytes(4091, id, 0), 750); + assert_eq!(database::owner_bytes(4091, 0, character), 1000); + // Another account's uploads are not on this one's bill + assert_eq!(database::owner_bytes(4092, 0, 0), 0); + + assert!(check_quota(4091, 1, 0, 0).is_ok()); + assert!(check_quota(4091, MAX_BYTES_PER_USER, 0, 0).unwrap_err().contains("per-account limit")); + + wipe(4091); + } + } diff --git a/src/router/custom_song.rs b/src/router/custom_song.rs index 76bdf41..c841d1a 100644 --- a/src/router/custom_song.rs +++ b/src/router/custom_song.rs @@ -47,6 +47,39 @@ const DEFAULT_BPM: f64 = 120.0; const DEFAULT_PREVIEW_LENGTH_SEC: f64 = 30.0; const PREVIEW_FADE_SEC: f64 = 0.5; +// Upload limits, enforced while the multipart field is still streaming (the 25MB +// PayloadConfig in lib.rs binds the String/Bytes extractors, not Multipart), and +// again over a package's expanded contents. The binding item is the audio track: +// 64MB holds a five-minute 44.1kHz stereo WAV or any realistic ogg/mp3, and the +// same figure is custom_3dmv's per-file cap. The per-request cap is twice that, +// which covers audio + jacket + four charts, or an export package plus the field +// map it expands into (the package field itself is removed before expansion) +pub const MAX_FILE_BYTES: usize = 64 * 1024 * 1024; +pub const MAX_REQUEST_BYTES: usize = 128 * 1024 * 1024; + +// The longest track that may be decoded. Enforced inside the decoder's packet +// loop, not after it: the decode accumulates planar f32 PCM, so an hour-long +// input is ~1.4GB of Vec before anything downstream gets to reject it. Official +// lives run 2-3 minutes; ten is already far past any real chart +pub const MAX_AUDIO_SECONDS: f64 = 600.0; + +// The largest jacket the image decoder is allowed to allocate for. Checked as a +// dimension/allocation limit BEFORE decode (image 0.25's own default is a 512MB +// allocation ceiling and no dimension bound at all), so a 40000x40000 png that +// compresses to a few KB is refused instead of decoded +const MAX_JACKET_DIM: u32 = 8192; +// The dimension cap is the binding one (8192^2 RGBA is 256MiB); this is the +// backstop for a decoder that wants scratch beyond the final buffer, and it sits +// below the crate's own 512MB default +const MAX_JACKET_ALLOC_BYTES: u64 = 384 * 1024 * 1024; + +// Per-account storage quota, counted over the sizes the catalog quotes to the +// client (both jackets, every chart, both audio cues). The original upload +// artifacts kept under original/ roughly double the on-disk figure, so 2GiB of +// catalog bytes is ~4GiB of disk - about two hundred songs, an order of magnitude +// past any real uploader, while keeping one account from filling the volume +pub const MAX_BYTES_PER_USER: i64 = 2 * 1024 * 1024 * 1024; + lazy_static! { // music_id assignment and the insert must not race between two uploads static ref UPLOAD_LOCK: Mutex<()> = Mutex::new(()); @@ -234,6 +267,15 @@ pub fn sweep_audio() { drop(lock); } +// The per-id asset files, for the webui's song pages (the game client fetches +// jackets and charts through /data/{md5} instead - the URLs this route serves are +// only ever followed by a browser that carries the webui session cookie). +// +// Unlike /data and /audio this route is addressed by a SEQUENTIAL id, not by an +// unguessable content hash, so the "the hash is the capability" argument that +// makes those two sessionless does not apply: without a visibility check the +// whole private and shared catalog's jackets and full chart JSON could be walked +// by anyone from music_id 10000 up. It gets the catalog's own rule async fn assets(req: HttpRequest) -> HttpResponse { if disabled() { return HttpResponse::NotFound().finish(); @@ -245,6 +287,10 @@ async fn assets(req: HttpRequest) -> HttpResponse { if music_id < database::FIRST_MUSIC_ID || !valid { return HttpResponse::NotFound().finish(); } + // A song the viewer may not see 404s rather than admitting it exists + if !database::asset_visible(music_id, get_session_uid(&req)) { + return HttpResponse::NotFound().finish(); + } match fs::read(song_path(music_id, &file)) { Ok(body) => { let mime = mime_guess::from_path(&file).first_or_octet_stream(); @@ -297,6 +343,17 @@ async fn data(req: HttpRequest) -> HttpResponse { return HttpResponse::NotFound().finish(); }; match fs::read(song_path(music_id, &filename)) { + // Jackets and charts live at FIXED per-song filenames that an in-place edit + // overwrites, so between the file write and the catalog update the index + // still points an old md5 at bytes that are no longer its own. The client + // caches whatever it downloads under the md5 it asked for and never + // re-checks, so serving those bytes would poison its cache permanently. + // The index is a hint; these bytes are the answer only if they hash to the + // request. A mismatch is the same 404 a stale md5 already gets, and the + // client re-downloads under the md5 the catalog now carries + Ok(body) if !hash.eq_ignore_ascii_case(&format!("{:x}", md5::compute(&body))) => { + HttpResponse::NotFound().finish() + }, Ok(body) => { let mime = mime_guess::from_path(&filename).first_or_octet_stream(); HttpResponse::Ok() @@ -320,19 +377,56 @@ fn send_json(resp: JsonValue) -> HttpResponse { .body(jzon::stringify(resp)) } +// The per-file cap is enforced while the field is still streaming, BEFORE any byte +// reaches the audio decoder, the png decoder or the chart parser. The per-request +// cap is checked over the running total async fn read_multipart(mut payload: Multipart) -> Result>, String> { let mut fields = HashMap::new(); + let mut total = 0usize; while let Some(mut field) = payload.try_next().await.map_err(|e| e.to_string())? { let name = field.name().unwrap_or("").to_string(); let mut data = Vec::new(); while let Some(chunk) = field.try_next().await.map_err(|e| e.to_string())? { + total += chunk.len(); + if total > MAX_REQUEST_BYTES { + return Err(over_request_limit()); + } data.extend_from_slice(&chunk); + if data.len() > MAX_FILE_BYTES { + return Err(over_file_limit(&name)); + } } fields.insert(name, data); } Ok(fields) } +pub fn over_file_limit(name: &str) -> String { + format!("'{}' exceeds the {} MB per-file limit", name, MAX_FILE_BYTES / (1024 * 1024)) +} + +pub fn over_request_limit() -> String { + format!("Upload exceeds the {} MB per-request limit", MAX_REQUEST_BYTES / (1024 * 1024)) +} + +// The same accounting read_multipart applies, re-run over a field map that came +// out of an export package. package::expand caps every entry as it inflates, but +// the caps have to hold over the RESULT too: a package is one multipart field and +// its expansion replaces the whole form +fn check_field_caps(fields: &HashMap>) -> Result<(), String> { + let mut total = 0usize; + for (name, data) in fields.iter() { + if data.len() > MAX_FILE_BYTES { + return Err(over_file_limit(name)); + } + total += data.len(); + if total > MAX_REQUEST_BYTES { + return Err(over_request_limit()); + } + } + Ok(()) +} + fn field_str(fields: &HashMap>, key: &str) -> String { String::from_utf8_lossy(fields.get(key).map(|v| v.as_slice()).unwrap_or(&[])).trim().to_string() } @@ -378,7 +472,21 @@ fn validate_shared_users(shared_with: &JsonValue) -> Result<(), String> { // Pad/crop the upload to a square, then resize to 512x512 fn process_jacket(bytes: &[u8]) -> Result<(Vec, Vec), String> { - let img = image::load_from_memory(bytes).map_err(|_| String::from("Jacket is not a valid png/jpg image"))?; + // Dimension and allocation limits BEFORE the decode: the header is read, the + // pixels are not, so a highly compressed enormous image is refused instead of + // being expanded into memory + let mut limits = image::Limits::default(); + limits.max_image_width = Some(MAX_JACKET_DIM); + limits.max_image_height = Some(MAX_JACKET_DIM); + limits.max_alloc = Some(MAX_JACKET_ALLOC_BYTES); + let mut reader = image::ImageReader::new(std::io::Cursor::new(bytes)) + .with_guessed_format() + .map_err(|_| String::from("Jacket is not a valid png/jpg image"))?; + reader.limits(limits); + let img = reader.decode().map_err(|e| match e { + image::ImageError::Limits(_) => format!("Jacket is larger than the {}x{} limit", MAX_JACKET_DIM, MAX_JACKET_DIM), + _ => String::from("Jacket is not a valid png/jpg image") + })?; let size = std::cmp::min(img.width(), img.height()); let jacket = img .crop_imm((img.width() - size) / 2, (img.height() - size) / 2, size, size) @@ -410,6 +518,18 @@ fn cue_json(cue: &audio::Cue, cue_name: String, is_loop: bool) -> JsonValue { } } +// A cue's stored md5, or None when the blob has no usable one. JsonValue::to_string +// renders Null as the literal "null", so a missing md5 used to arrive at the GC as +// a five-character string that passed an is_empty() guard; only exactly 32 hex +// characters is a hash (the same test the startup sweep and custom_3dmv's GC use) +fn cue_md5(cue: &JsonValue) -> Option { + let md5 = cue["md5"].as_str()?; + if md5.len() != 32 || !md5.chars().all(|c| c.is_ascii_hexdigit()) { + return None; + } + Some(md5.to_string()) +} + // (md5-hex, byte-length) of a downloadable asset's served bytes. The client // caches charts/jackets content-addressed by this md5, so it must be the hash // of the exact bytes the data route returns @@ -658,6 +778,8 @@ fn create_song(uid: i64, fields: &HashMap>) -> Result>) -> Result Result<(), String> { + let used = database::owner_bytes(uid, excluded_music_id); + if used + adding > MAX_BYTES_PER_USER { + return Err(format!( + "This upload would put your songs at {} MB, over the {} MB per-account limit - delete a song first", + (used + adding) / (1024 * 1024), MAX_BYTES_PER_USER / (1024 * 1024) + )); + } + Ok(()) +} + // Edit an existing song in place. The music_id - and everything derived from // it: live_id, cue names, note_data_file_name, asset URLs - stays the same, so // player score records survive (delete + re-upload retires the id and wipes @@ -924,6 +1060,11 @@ fn update_song(music_id: i64, fields: &HashMap>) -> Result<(), S "sound": sound }; + // The resulting song has to fit the uploader's quota, with the stored copy of + // this same song excluded (it is being replaced, not added to) + let owner = database::get_song_owner(music_id).ok_or(String::from("Song not found"))?; + check_quota(owner, database::song_bytes(&song), music_id)?; + // Same serialization as upload around the writes and the revision bump let lock = lock_onto_mutex!(UPLOAD_LOCK); if let (Some((jacket, jacket_blur)), Some(bytes)) = (&jacket, jacket_bytes) { @@ -954,17 +1095,21 @@ fn update_song(music_id: i64, fields: &HashMap>) -> Result<(), S database::update_song(music_id, &song); database::bump_revision(); - drop(lock); // Replaced cues: the old oggs are content-addressed and may be shared with - // another song (or unchanged by this edit) - GC them the same way delete does - let kept = [song["sound"]["play"]["md5"].to_string(), song["sound"]["select"]["md5"].to_string()]; + // another song (or unchanged by this edit) - GC them the same way delete does. + // INSIDE the lock, for the reason delete states: an upload writes its oggs + // before it inserts its row, so a GC that reads the catalog in that window + // sees no reference to a shared md5 and unlinks the file the new song is + // about to serve. A read error means "assume referenced" - never unlink + let kept: Vec = ["play", "select"].iter().filter_map(|key| cue_md5(&song["sound"][*key])).collect(); for key in ["play", "select"] { - let md5 = old_song["sound"][key]["md5"].to_string(); - if !md5.is_empty() && !kept.contains(&md5) && !database::audio_in_use(&md5, music_id) { + let Some(md5) = cue_md5(&old_song["sound"][key]) else { continue; }; + if !kept.contains(&md5) && !database::audio_in_use(&md5, music_id).unwrap_or(true) { let _ = fs::remove_file(audio_file_path(&md5)); } } + drop(lock); Ok(()) } @@ -980,21 +1125,29 @@ async fn upload(req: HttpRequest, payload: Multipart) -> HttpResponse { Ok(fields) => fields, Err(e) => return webui::error(&e) }; - // An export package from another server: its contents map 1:1 onto the - // normal upload fields, so importing is just an upload - if let Some(bytes) = fields.remove("package") { - if !bytes.is_empty() { - if let Err(e) = package::expand(&bytes, &mut fields) { - return webui::error(&e); + // Everything from here on is seconds of CPU - zip inflation, a png decode and + // resize, a whole-track vorbis encode - so it runs on the blocking pool + // instead of the actix worker that has to keep serving the game API + let result = web::block(move || { + // An export package from another server: its contents map 1:1 onto the + // normal upload fields, so importing is just an upload + if let Some(bytes) = fields.remove("package") { + if !bytes.is_empty() { + package::expand(&bytes, &mut fields)?; + // The expansion replaced the form: it has to satisfy the same + // per-file/per-request caps the multipart reader enforces + check_field_caps(&fields)?; } } - } - match create_song(uid, &fields) { - Ok(music_id) => send_json(object!{ + create_song(uid, &fields) + }).await; + match result { + Ok(Ok(music_id)) => send_json(object!{ result: "OK", music_id: music_id }), - Err(e) => webui::error(&e) + Ok(Err(e)) => webui::error(&e), + Err(_) => webui::error("The upload could not be processed") } } @@ -1018,12 +1171,15 @@ async fn update(req: HttpRequest, payload: Multipart) -> HttpResponse { if owner != uid { return webui::error("You can only manage your own songs"); } - match update_song(music_id, &fields) { - Ok(()) => send_json(object!{ + // Same blocking-pool treatment as upload: an edit can re-encode the audio and + // re-derive the jackets + match web::block(move || update_song(music_id, &fields)).await { + Ok(Ok(())) => send_json(object!{ result: "OK", music_id: music_id }), - Err(e) => webui::error(&e) + Ok(Err(e)) => webui::error(&e), + Err(_) => webui::error("The edit could not be processed") } } @@ -1107,7 +1263,9 @@ async fn visibility(req: HttpRequest, body: String) -> HttpResponse { return webui::error(&e); } - database::set_visibility(music_id, &visibility, &shared_with); + if let Err(e) = database::set_visibility(music_id, &visibility, &shared_with) { + return webui::error(&format!("Could not change the visibility: {}", e)); + } // The download toggle only affects the webui browser, not the game catalog if !body["downloads_disabled"].is_null() { database::set_downloads_disabled(music_id, body["downloads_disabled"].as_bool().unwrap_or(false)); @@ -1142,7 +1300,9 @@ async fn delete(req: HttpRequest, body: String) -> HttpResponse { // catalog in that window sees no reference to a shared md5 and would // unlink the file the new song is about to serve let lock = lock_onto_mutex!(UPLOAD_LOCK); - database::delete_song(music_id); + if let Err(e) = database::delete_song(music_id) { + return webui::error(&format!("Could not delete the song: {}", e)); + } database::bump_revision(); // Global clear-rate stats for the dead live id (per-user score records are // wiped lazily on each user's next userdata pull) @@ -1151,10 +1311,11 @@ async fn delete(req: HttpRequest, body: String) -> HttpResponse { crate::router::custom_3dmv::purge_song(music_id); let _ = fs::remove_dir_all(get_data_path(&format!("custom_songs/{}", music_id))); - // Audio is content-addressed and may be shared with another upload + // Audio is content-addressed and may be shared with another upload. A read + // error means "assume referenced" - never unlink on a doubtful reference set for key in ["play", "select"] { - let md5 = song["sound"][key]["md5"].to_string(); - if !md5.is_empty() && !database::audio_in_use(&md5, music_id) { + let Some(md5) = cue_md5(&song["sound"][key]) else { continue; }; + if !database::audio_in_use(&md5, music_id).unwrap_or(true) { let _ = fs::remove_file(audio_file_path(&md5)); } } @@ -1167,6 +1328,36 @@ async fn delete(req: HttpRequest, body: String) -> HttpResponse { +// Every song this account uploaded, gone - called from userdata::delete_account, +// so a purged uploader leaves no catalog row pointing at a user id that no longer +// resolves (browse renders an uploader name for every row). Runs the same steps +// the owner's own delete does, including the cross-feature MV cascade and the +// content-addressed audio GC +pub fn purge_owner(uid: i64) { + if disabled() { + return; + } + for music_id in database::music_ids_by_owner(uid) { + let song = database::get_song(music_id).unwrap_or(object!{}); + let lock = lock_onto_mutex!(UPLOAD_LOCK); + if database::delete_song(music_id).is_err() { + drop(lock); + continue; + } + database::bump_revision(); + crate::router::clear_rate::purge_live(music_id); + crate::router::custom_3dmv::purge_song(music_id); + let _ = fs::remove_dir_all(get_data_path(&format!("custom_songs/{}", music_id))); + for key in ["play", "select"] { + let Some(md5) = cue_md5(&song["sound"][key]) else { continue; }; + if !database::audio_in_use(&md5, music_id).unwrap_or(true) { + let _ = fs::remove_file(audio_file_path(&md5)); + } + } + drop(lock); + } +} + /// WHY DID THE AI WRITE 400 LINES OF TESTS /// Well I guess they can't hurt lets commit them anyway @@ -1844,11 +2035,11 @@ mod tests { let stranger = 5555; let public_id = database::next_music_id(); - database::insert_song(public_id, owner, &object!{music_id: public_id}, "public", &array![], false); + database::insert_song(public_id, owner, &object!{music_id: public_id}, "public", &array![], false).unwrap(); let private_id = database::next_music_id(); - database::insert_song(private_id, owner, &object!{music_id: private_id}, "private", &array![], false); + database::insert_song(private_id, owner, &object!{music_id: private_id}, "private", &array![], false).unwrap(); let shared_id = database::next_music_id(); - database::insert_song(shared_id, owner, &object!{music_id: shared_id}, "shared", &array![friend], false); + database::insert_song(shared_id, owner, &object!{music_id: shared_id}, "shared", &array![friend], false).unwrap(); let has = |songs: &JsonValue, id: i64| songs.members().any(|data| data["music_id"] == id); @@ -1870,11 +2061,11 @@ mod tests { let stranger = 7777; let locked_id = database::next_music_id(); - database::insert_song(locked_id, owner, &object!{music_id: locked_id}, "public", &array![], true); + database::insert_song(locked_id, owner, &object!{music_id: locked_id}, "public", &array![], true).unwrap(); let open_id = database::next_music_id(); - database::insert_song(open_id, owner, &object!{music_id: open_id}, "public", &array![], false); + database::insert_song(open_id, owner, &object!{music_id: open_id}, "public", &array![], false).unwrap(); let private_id = database::next_music_id(); - database::insert_song(private_id, owner, &object!{music_id: private_id}, "private", &array![], false); + database::insert_song(private_id, owner, &object!{music_id: private_id}, "private", &array![], false).unwrap(); // Downloads disabled: everyone but the owner is denied assert!(database::export_allowed(locked_id, Some(owner)).is_ok()); @@ -1976,7 +2167,7 @@ mod tests { let owner = 4242; let music_id = database::next_music_id(); - database::insert_song(music_id, owner, &object!{music_id: music_id}, "public", &array![], false); + database::insert_song(music_id, owner, &object!{music_id: music_id}, "public", &array![], false).unwrap(); // Sanity: the feature is on, so the id is visible to get_music_ids assert!(get_music_ids(owner).contains(music_id)); @@ -2114,12 +2305,12 @@ mod tests { music_id: public_id, name: "Public & \"Co\"", name_en: "Public Song EN" - }, "public", &array![], false); + }, "public", &array![], false).unwrap(); let private_id = database::next_music_id(); database::insert_song(private_id, 6100, &object!{ music_id: private_id, name: "Top Secret Anthem" - }, "private", &array![], false); + }, "private", &array![], false).unwrap(); for id in [public_id, private_id] { clear_rate::live_completed(id, 1, false, 100, 6100); } @@ -2162,11 +2353,11 @@ mod tests { let outsider = 5003; let public_id = database::next_music_id(); - database::insert_song(public_id, owner, &object!{music_id: public_id}, "public", &array![], false); + database::insert_song(public_id, owner, &object!{music_id: public_id}, "public", &array![], false).unwrap(); let private_id = database::next_music_id(); - database::insert_song(private_id, owner, &object!{music_id: private_id}, "private", &array![], false); + database::insert_song(private_id, owner, &object!{music_id: private_id}, "private", &array![], false).unwrap(); let shared_id = database::next_music_id(); - database::insert_song(shared_id, owner, &object!{music_id: shared_id}, "shared", &array![shared_user], false); + database::insert_song(shared_id, owner, &object!{music_id: shared_id}, "shared", &array![shared_user], false).unwrap(); // A stock live id, outside the custom range - never filtered let stock_id: i64 = 1_500_123; @@ -2201,4 +2392,433 @@ mod tests { assert!(!sees(uid, private_id) && !sees(uid, shared_id)); } } + + // ---- defect-fix coverage ------------------------------------------------- + + use actix_web::test::TestRequest; + use std::io::Write; + + // A real multipart body, so the streaming caps in read_multipart are exercised + // by the reader itself rather than by a hand-built field map + async fn multipart_of(parts: Vec<(&str, Vec)>) -> Multipart { + let boundary = "ewtestboundary"; + let mut body: Vec = Vec::new(); + for (name, data) in parts { + body.extend(format!( + "--{}\r\nContent-Disposition: form-data; name=\"{}\"; filename=\"{}\"\r\nContent-Type: application/octet-stream\r\n\r\n", + boundary, name, name + ).into_bytes()); + body.extend(data); + body.extend(b"\r\n"); + } + body.extend(format!("--{}--\r\n", boundary).into_bytes()); + let (req, mut payload) = TestRequest::default() + .insert_header(("content-type", format!("multipart/form-data; boundary={}", boundary))) + .set_payload(actix_web::web::Bytes::from(body)) + .to_http_parts(); + ::from_request(&req, &mut payload).await.unwrap() + } + + // A real webui session for `uid`, the way the browser gets one + fn webui_session(auth_token: &str) -> (i64, String) { + let uid = userdata::get_acc(auth_token)["user"]["id"].as_i64().unwrap(); + userdata::user::migration::save_acc_transfer(uid, "hunter2"); + (uid, userdata::webui_login(uid, "hunter2").unwrap()) + } + + // A jacket whose processed bytes are unique to this seed. The md5 index is + // content-addressed ACROSS songs, so a shared test_png() would let one test's + // jacket md5 resolve to another test's file + fn seeded_png(seed: u8) -> Vec { + let mut rv = Vec::new(); + image::DynamicImage::ImageRgba8(image::RgbaImage::from_fn(64, 32, |x, y| { + image::Rgba([(x * 4) as u8, (y * 8) as u8, seed, 255]) + })).write_to(&mut std::io::Cursor::new(&mut rv), image::ImageFormat::Png).unwrap(); + rv + } + + fn song_fields(name: &str, tone: f32) -> HashMap> { + let mut fields = HashMap::new(); + field(&mut fields, "name", name); + field(&mut fields, "artist", "A"); + field(&mut fields, "attribute", "1"); + fields.insert(String::from("jacket"), seeded_png(tone as u8)); + fields.insert(String::from("audio"), test_ogg_tone(tone)); + fields.insert(String::from("chart_1"), test_chart()); + fields + } + + // Renaming the table away is the cheapest way to make every query against it + // fail the way a busy/corrupt database does, without losing the rows + fn with_songs_table_broken(body: impl FnOnce() -> T) -> T { + let conn = rusqlite::Connection::open(database::test_db_path()).unwrap(); + conn.execute("ALTER TABLE songs RENAME TO songs_hidden", ()).unwrap(); + let rv = body(); + conn.execute("ALTER TABLE songs_hidden RENAME TO songs", ()).unwrap(); + rv + } + + // D6: the multipart reader caps a field WHILE it streams, before any byte + // reaches the audio/png/chart parsers. Every other test builds the field map + // directly, so this is the only one that goes through the reader itself + #[test] + fn the_multipart_reader_caps_an_oversize_field() { + let _lock = crate::runtime::lock_test_data_path(); + actix_web::rt::System::new().block_on(async { + let oversize = vec![b'a'; MAX_FILE_BYTES + 1]; + let err = read_multipart(multipart_of(vec![("audio", oversize)]).await).await.unwrap_err(); + assert!(err.contains("'audio'") && err.contains("per-file limit"), "{}", err); + + // A body inside the caps still arrives intact + let fields = read_multipart(multipart_of(vec![ + ("name", b"Capped".to_vec()), + ("jacket", test_png()) + ]).await).await.unwrap(); + assert_eq!(field_str(&fields, "name"), "Capped"); + assert_eq!(fields.get("jacket"), Some(&test_png())); + }); + } + + // D1/D6: the per-request total is accounted over the whole form, which is also + // what a package's expanded contents are re-checked against + #[test] + fn the_request_total_is_capped() { + let mut fields: HashMap> = HashMap::new(); + fields.insert(String::from("a"), vec![0; MAX_FILE_BYTES]); + fields.insert(String::from("b"), vec![0; MAX_FILE_BYTES]); + assert!(check_field_caps(&fields).is_ok()); + fields.insert(String::from("c"), vec![0; 1]); + let err = check_field_caps(&fields).unwrap_err(); + assert!(err.contains("per-request limit"), "{}", err); + + let mut one = HashMap::new(); + one.insert(String::from("audio"), vec![0; MAX_FILE_BYTES + 1]); + assert!(check_field_caps(&one).unwrap_err().contains("per-file limit")); + } + + // D1: a package entry is capped as it inflates. Deflate's ~1032:1 ceiling + // means an uncapped read_to_end here turns a tiny zip into gigabytes + #[test] + fn package_import_is_capped() { + let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + let options = zip::write::SimpleFileOptions::default(); + zip.start_file("manifest.json", options).unwrap(); + zip.write_all(jzon::stringify(object!{ + "format": 1, "name": "Bomb", "artist": "A", "attribute": 1, + "levels": [{ "level": 1, "level_number": 3 }] + }).as_bytes()).unwrap(); + zip.start_file("jacket", options).unwrap(); + zip.write_all(&test_png()).unwrap(); + zip.start_file("chart_1.json", options).unwrap(); + zip.write_all(&test_chart()).unwrap(); + // Compresses to a few KB, inflates to just over the per-file cap + zip.start_file("audio", options).unwrap(); + zip.write_all(&vec![0u8; MAX_FILE_BYTES + 1]).unwrap(); + let package = zip.finish().unwrap().into_inner(); + assert!(package.len() < 1024 * 1024, "the bomb should be small: {} bytes", package.len()); + + let mut fields = HashMap::new(); + let err = package::expand(&package, &mut fields).unwrap_err(); + assert!(err.contains("per-file limit"), "{}", err); + // Nothing oversized was buffered into the field map + assert!(fields.get("audio").is_none()); + } + + // D3: /custom_song/assets/{music_id}/{file} is addressed by a SEQUENTIAL id, + // not by a content hash, so it gets the catalog's visibility rule: the whole + // private/shared catalog used to be walkable from music_id 10000 up + #[test] + fn the_asset_route_is_visibility_gated() { + let _lock = crate::runtime::lock_test_data_path(); + let (owner, owner_cookie) = webui_session("custom-song-assets-owner"); + let (_, stranger_cookie) = webui_session("custom-song-assets-stranger"); + + let public_id = create_song(owner, &song_fields("Assets Public", 331.0)).unwrap(); + let private_id = create_song(owner, &song_fields("Assets Private", 337.0)).unwrap(); + database::set_visibility(private_id, "private", &array![]).unwrap(); + + let call = |music_id: i64, cookie: Option<&str>| -> HttpResponse { + let mut req = TestRequest::default() + .param("music_id", music_id.to_string()) + .param("file", String::from("jacket.png")); + if let Some(cookie) = cookie { + req = req.insert_header(("Cookie", format!("ew_token={}", cookie))); + } + let req = req.to_http_request(); + actix_web::rt::System::new().block_on(async { assets(req).await }) + }; + let ok = actix_web::http::StatusCode::OK; + let missing = actix_web::http::StatusCode::NOT_FOUND; + + // Public: everyone, session or not + assert_eq!(call(public_id, None).status(), ok); + assert_eq!(call(public_id, Some(&stranger_cookie)).status(), ok); + // Private: the owner only + assert_eq!(call(private_id, Some(&owner_cookie)).status(), ok); + assert_eq!(call(private_id, None).status(), missing); + assert_eq!(call(private_id, Some(&stranger_cookie)).status(), missing); + + // Shared: the owner plus the shared list + let stranger = userdata::get_acc(&userdata::webui_login_token(&stranger_cookie).unwrap())["user"]["id"].as_i64().unwrap(); + database::set_visibility(private_id, "shared", &array![stranger]).unwrap(); + assert_eq!(call(private_id, Some(&stranger_cookie)).status(), ok); + assert_eq!(call(private_id, None).status(), missing); + + purge_owner(owner); + } + + // D14: jackets and charts live at fixed per-song filenames that an in-place + // edit overwrites, so the md5 index can briefly point at bytes that are no + // longer its own. The client caches by md5 and never re-checks, so the route + // verifies before it serves + #[test] + fn the_data_route_refuses_bytes_that_do_not_match_the_md5() { + let _lock = crate::runtime::lock_test_data_path(); + let music_id = create_song(7710, &song_fields("Mismatch", 349.0)).unwrap(); + let jacket_md5 = database::get_song(music_id).unwrap()["jacket_md5"].to_string(); + + let call = || -> HttpResponse { + let req = TestRequest::default() + .param("hash", jacket_md5.clone()) + .param("file", format!("{}.png", jacket_md5)) + .to_http_request(); + actix_web::rt::System::new().block_on(async { data(req).await }) + }; + assert_eq!(call().status(), actix_web::http::StatusCode::OK); + + // The index still resolves, but the file no longer holds those bytes + let real = fs::read(song_path(music_id, "jacket.png")).unwrap(); + let mut different = real.clone(); + different.extend(b"not the bytes that hash to that md5"); + fs::write(song_path(music_id, "jacket.png"), &different).unwrap(); + assert_eq!(call().status(), actix_web::http::StatusCode::NOT_FOUND); + + fs::write(song_path(music_id, "jacket.png"), &real).unwrap(); + assert_eq!(call().status(), actix_web::http::StatusCode::OK); + purge_owner(7710); + } + + // D5: the chart transcoder is linear per note and its input is only bounded in + // bytes, so the note count has its own ceiling - and the duplicate check that + // used to rescan every preceding note still rejects what it always did + #[test] + fn chart_size_is_bounded_and_duplicates_still_rejected() { + let mut too_many = jzon::array![]; + for i in 0..(chart::MAX_NOTES + 1) { + too_many.push(object!{ + "timing_sec": 1.0 + i as f64 * 0.001, "notes_attribute": 1, "notes_level": 1, + "effect": 1, "effect_value": 0.0, "position": 5 + }).unwrap(); + } + let err = chart::transcode(&too_many).unwrap_err(); + assert!(err.contains("the maximum is"), "{}", err); + + // A large but legal chart still transcodes (and does so in linear time) + let mut big = jzon::array![]; + for i in 0..5000 { + big.push(object!{ + "timing_sec": 1.0 + i as f64 * 0.01, "notes_attribute": 1, "notes_level": 1, + "effect": 1, "effect_value": 0.0, "position": (i % 9) + 1 + }).unwrap(); + } + let (_, combo) = chart::transcode(&big).unwrap(); + assert_eq!(combo, 5000); + + // Same timing + position with a different effect is still a rejection + let clash = jzon::array![ + {"timing_sec": 1.0, "notes_attribute": 1, "notes_level": 1, "effect": 1, "effect_value": 0.0, "position": 4}, + {"timing_sec": 1.0, "notes_attribute": 1, "notes_level": 1, "effect": 3, "effect_value": 0.5, "position": 4} + ]; + assert!(chart::transcode(&clash).unwrap_err().contains("duplicate timing")); + + // ...and a non-finite timing is refused instead of panicking the + // time-order sort's partial_cmp().unwrap() + let nan = jzon::array![ + {"timing_sec": f64::INFINITY, "notes_attribute": 1, "notes_level": 1, "effect": 1, "effect_value": 0.0, "position": 4} + ]; + assert!(chart::transcode(&nan).unwrap_err().contains("finite")); + } + + // D2/D8: a read error must never read as "no rows". The GC unlinks on + // "not referenced", so a failed lookup has to mean "assume referenced" - and a + // failed write has to reach the uploader as an error, not as a worker panic + #[test] + fn a_database_error_never_deletes_audio_or_panics() { + let _lock = crate::runtime::lock_test_data_path(); + let music_id = create_song(7711, &song_fields("Fail Closed", 353.0)).unwrap(); + let play = database::get_song(music_id).unwrap()["sound"]["play"]["md5"].to_string(); + assert_eq!(database::audio_in_use(&play, 0), Ok(true)); + assert_eq!(database::audio_in_use(&"c3".repeat(16), 0), Ok(false)); + + with_songs_table_broken(|| { + // Fail-closed: an error, not a "false" that would unlink a live file + assert!(database::audio_in_use(&play, 0).is_err()); + assert!(database::audio_in_use(&play, 0).unwrap_or(true)); + + // And an insert that cannot land is an error response, not a panic + let err = create_song(7711, &song_fields("Never Stored", 359.0)).unwrap_err(); + assert!(err.contains("Could not store the song"), "{}", err); + }); + + assert!(fs::read(audio_file_path(&play)).is_ok(), "the live ogg was unlinked"); + purge_owner(7711); + } + + // D15: the quota counts the bytes the catalog quotes to the client, per owner + #[test] + fn uploads_are_bounded_by_a_per_account_byte_quota() { + let _lock = crate::runtime::lock_test_data_path(); + purge_owner(7712); + assert_eq!(database::owner_bytes(7712, 0), 0); + + let music_id = create_song(7712, &song_fields("Quota", 367.0)).unwrap(); + let song = database::get_song(music_id).unwrap(); + let stored = database::song_bytes(&song); + assert!(stored > 0); + assert_eq!(database::owner_bytes(7712, 0), stored); + // An in-place edit replaces its own bytes rather than adding to them + assert_eq!(database::owner_bytes(7712, music_id), 0); + + assert!(check_quota(7712, 1, 0).is_ok()); + let err = check_quota(7712, MAX_BYTES_PER_USER, 0).unwrap_err(); + assert!(err.contains("per-account limit"), "{}", err); + // Another account's uploads are not on this one's bill + assert_eq!(database::owner_bytes(7713, 0), 0); + + purge_owner(7712); + } + + // D10: purging an account takes its uploads in all three features with it - + // otherwise every catalog keeps rows whose owner_id no longer resolves, and + // browse renders an uploader name for each of them + #[test] + fn deleting_an_account_purges_its_uploads() { + let _lock = crate::runtime::lock_test_data_path(); + let auth = "custom-content-purge-token"; + let uid = userdata::get_acc(auth)["user"]["id"].as_i64().unwrap(); + + let music_id = create_song(uid, &song_fields("Purged", 373.0)).unwrap(); + let dir = get_data_path(&format!("custom_songs/{}", music_id)); + let play = database::get_song(music_id).unwrap()["sound"]["play"]["md5"].to_string(); + + let mv_id = crate::database::custom_3dmv::next_mv_id(); + crate::database::custom_3dmv::insert_mv(mv_id, music_id, uid, &object!{ + "mv_id": mv_id, "music_id": music_id, "name": "Purged MV", "member_count": 1, "files": [] + }, true).unwrap(); + let card_id = crate::database::custom_card::next_card_id(); + crate::database::custom_card::insert_card(card_id, 1001, uid, &object!{ + "master_card_id": card_id, "rarity": 1 + }, true, true).unwrap(); + let character_id = crate::database::custom_card::next_character_id(); + crate::database::custom_card::insert_character(character_id, uid, &object!{ + "master_character_id": character_id, "name": "Purged" + }).unwrap(); + + userdata::delete_account(uid); + + assert!(database::get_song(music_id).is_none(), "the song survived the purge"); + assert!(fs::metadata(&dir).is_err(), "the song's files survived the purge"); + assert!(fs::read(audio_file_path(&play)).is_err(), "the song's audio survived the purge"); + assert!(crate::database::custom_3dmv::get_mv(mv_id).is_none(), "the MV survived the purge"); + assert!(crate::database::custom_card::get_card(card_id).is_none(), "the card survived the purge"); + assert!(crate::database::custom_card::get_character(character_id).is_none(), "the character survived the purge"); + } + + + // D1/D6/D7: the whole upload route, end to end - a real multipart body, a real + // session, the expansion of a real package, and the create running on the + // blocking pool instead of on the actix worker + #[test] + fn the_upload_route_runs_end_to_end_under_its_caps() { + let _lock = crate::runtime::lock_test_data_path(); + let (uid, cookie) = webui_session("custom-song-upload-route"); + purge_owner(uid); + + let post = |parts: Vec<(&'static str, Vec)>| -> String { + let cookie = cookie.clone(); + actix_web::rt::System::new().block_on(async move { + let payload = multipart_of(parts).await; + let req = TestRequest::default() + .insert_header(("Cookie", format!("ew_token={}", cookie))) + .to_http_request(); + let resp = upload(req, payload).await; + let bytes = actix_web::body::to_bytes(resp.into_body()).await.unwrap(); + String::from_utf8_lossy(&bytes).to_string() + }) + }; + + // A normal upload lands + let body = post(vec![ + ("name", b"Route Song".to_vec()), + ("artist", b"A".to_vec()), + ("attribute", b"1".to_vec()), + ("jacket", seeded_png(211)), + ("audio", test_ogg_tone(211.0)), + ("chart_1", test_chart()) + ]); + let music_id = jzon::parse(&body).unwrap()["music_id"].as_i64().unwrap_or(0); + assert!(music_id >= database::FIRST_MUSIC_ID, "{}", body); + + // Its own export package re-imports through the same route + let package = package::build(music_id).unwrap(); + let body = post(vec![("package", package)]); + assert!(jzon::parse(&body).unwrap()["music_id"].as_i64().unwrap_or(0) > music_id, "{}", body); + + // An oversized field never reaches the decoders + let body = post(vec![("audio", vec![b'a'; MAX_FILE_BYTES + 1])]); + assert!(body.contains("per-file limit"), "{}", body); + + // ...and neither does one that only appears once the package is expanded + let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + let options = zip::write::SimpleFileOptions::default(); + zip.start_file("manifest.json", options).unwrap(); + zip.write_all(jzon::stringify(object!{ + "format": 1, "name": "Bomb", "artist": "A", "attribute": 1, + "levels": [{ "level": 1, "level_number": 3 }] + }).as_bytes()).unwrap(); + zip.start_file("jacket", options).unwrap(); + zip.write_all(&seeded_png(212)).unwrap(); + zip.start_file("audio", options).unwrap(); + zip.write_all(&vec![0u8; MAX_FILE_BYTES + 1]).unwrap(); + zip.start_file("chart_1.json", options).unwrap(); + zip.write_all(&test_chart()).unwrap(); + let body = post(vec![("package", zip.finish().unwrap().into_inner())]); + assert!(body.contains("per-file limit"), "{}", body); + + purge_owner(uid); + } + + // D4/D13: replacing a cue collects the ogg only that song referenced, and + // leaves one another song still names. The GC runs INSIDE the upload lock, so + // it cannot observe an upload that has written its oggs but not yet its row + #[test] + fn a_replaced_cue_is_collected_and_a_shared_one_is_kept() { + let _lock = crate::runtime::lock_test_data_path(); + purge_owner(7714); + purge_owner(7715); + + let shared = create_song(7714, &song_fields("Shared Audio A", 379.0)).unwrap(); + let other = create_song(7715, &song_fields("Shared Audio B", 379.0)).unwrap(); + let play = database::get_song(shared).unwrap()["sound"]["play"]["md5"].to_string(); + assert_eq!(database::get_song(other).unwrap()["sound"]["play"]["md5"].to_string(), play); + + // Replace the first song's audio: its old cue is still the second's + let mut edit = HashMap::new(); + edit.insert(String::from("audio"), test_ogg_tone(383.0)); + update_song(shared, &edit).unwrap(); + let replaced = database::get_song(shared).unwrap()["sound"]["play"]["md5"].to_string(); + assert_ne!(replaced, play); + assert!(fs::read(audio_file_path(&play)).is_ok(), "an ogg another song still serves was unlinked"); + assert!(fs::read(audio_file_path(&replaced)).is_ok()); + + // Now nothing else references it: replacing it again collects it + let mut edit = HashMap::new(); + edit.insert(String::from("audio"), test_ogg_tone(389.0)); + update_song(shared, &edit).unwrap(); + assert!(fs::read(audio_file_path(&replaced)).is_err(), "the orphaned ogg was kept"); + + purge_owner(7714); + purge_owner(7715); + } + } diff --git a/src/router/custom_song/audio.rs b/src/router/custom_song/audio.rs index d406148..6d7f303 100644 --- a/src/router/custom_song/audio.rs +++ b/src/router/custom_song/audio.rs @@ -7,7 +7,7 @@ use symphonia::core::formats::probe::Hint; use symphonia::core::io::MediaSourceStream; use vorbis_rs::{VorbisBitrateManagementStrategy, VorbisEncoderBuilder}; -use super::{DEFAULT_PREVIEW_LENGTH_SEC, PREVIEW_FADE_SEC}; +use super::{DEFAULT_PREVIEW_LENGTH_SEC, MAX_AUDIO_SECONDS, PREVIEW_FADE_SEC}; // The whole audio pipeline runs in-process: symphonia (pure Rust) decodes and // validates uploads, vorbis_rs (libvorbis compiled into the binary - a library @@ -53,7 +53,12 @@ fn is_ogg_vorbis(bytes: &[u8]) -> bool { bytes.starts_with(b"OggS") && bytes.len() > 64 && bytes[..64].windows(7).any(|w| w == b"\x01vorbis") } -fn decode(bytes: &[u8]) -> Result { +// `max_duration_sec` is enforced INSIDE the packet loop, not after it: the decode +// accumulates full planar f32 PCM, so checking the duration afterwards means the +// allocation has already happened (an hour of 44.1kHz stereo is ~1.4GB of Vec). +// Bailing at the first packet past the limit keeps the peak proportional to the +// limit instead of to the upload +fn decode(bytes: &[u8], max_duration_sec: f64) -> Result { let stream = MediaSourceStream::new(Box::new(std::io::Cursor::new(bytes.to_vec())), Default::default()); let mut format = symphonia::default::get_probe() .probe(&Hint::new(), stream, Default::default(), Default::default()) @@ -96,6 +101,9 @@ fn decode(bytes: &[u8]) -> Result { for (i, samples) in channels.iter_mut().enumerate() { samples.extend(interleaved.iter().skip(i).step_by(count)); } + if sample_rate > 0 && channels[0].len() as f64 / sample_rate as f64 > max_duration_sec { + return Err(format!("Audio is over the {:.0} second maximum", max_duration_sec)); + } } if channels.is_empty() || channels[0].is_empty() || sample_rate == 0 { @@ -141,14 +149,13 @@ fn cue(bytes: Vec, duration_sec: f64) -> Cue { // reads, keep it as-is when it's already ogg-vorbis, otherwise transcode. No // cuts, fades, loop points or preview split - mono or stereo as-sourced pub fn process_one_shot(bytes: &[u8], max_duration_sec: f64) -> Result { - let audio = decode(bytes)?; + // The length limit belongs to the decoder, which stops at the first packet + // past it rather than accumulating the whole clip and rejecting it afterwards + let audio = decode(bytes, max_duration_sec)?; let duration = audio.duration(); if duration < 0.2 { return Err(String::from("Audio clip is shorter than 0.2 seconds")); } - if duration > max_duration_sec { - return Err(format!("Audio clip is {:.1} seconds long - the maximum is {:.0} seconds", duration, max_duration_sec)); - } if is_ogg_vorbis(bytes) { return Ok(cue(bytes.to_vec(), duration)); } @@ -160,7 +167,7 @@ pub fn process_one_shot(bytes: &[u8], max_duration_sec: f64) -> Result, preview_length_sec: Option) -> Result<(Cue, Cue), String> { - let audio = decode(bytes)?; + let audio = decode(bytes, MAX_AUDIO_SECONDS)?; let duration = audio.duration(); if duration <= 1.0 { return Err(String::from("Audio track is too short")); @@ -200,3 +207,54 @@ pub fn process(bytes: &[u8], preview_start_sec: Option, preview_length_sec: Ok((play, select)) } + +#[cfg(test)] +mod tests { + use super::*; + + // 44.1kHz 16-bit mono, `seconds` long + fn wav(seconds: f64) -> Vec { + let sample_rate: u32 = 44100; + let frames = (seconds * sample_rate as f64) as u32; + let data_len = frames * 2; + let mut rv = Vec::new(); + rv.extend(b"RIFF"); + rv.extend((36 + data_len).to_le_bytes()); + rv.extend(b"WAVEfmt "); + rv.extend(16u32.to_le_bytes()); + rv.extend(1u16.to_le_bytes()); + rv.extend(1u16.to_le_bytes()); + rv.extend(sample_rate.to_le_bytes()); + rv.extend((sample_rate * 2).to_le_bytes()); + rv.extend(2u16.to_le_bytes()); + rv.extend(16u16.to_le_bytes()); + rv.extend(b"data"); + rv.extend(data_len.to_le_bytes()); + for i in 0..frames { + let sample = ((i as f64 * 440.0 * 2.0 * std::f64::consts::PI / sample_rate as f64).sin() * 8000.0) as i16; + rv.extend(sample.to_le_bytes()); + } + rv + } + + // The length limit is enforced from INSIDE the packet loop: the decode + // accumulates full planar f32 PCM, so a post-decode check means the + // allocation already happened (an hour of stereo is ~1.4GB of Vec) + #[test] + fn the_decoder_stops_at_the_duration_cap() { + let three_seconds = wav(3.0); + let audio = decode(&three_seconds, 5.0).unwrap(); + assert!((audio.duration() - 3.0).abs() < 0.01); + + let Err(err) = decode(&three_seconds, 1.0) else { + panic!("a three-second track decoded under a one-second cap"); + }; + assert!(err.contains("1 second maximum"), "{}", err); + // The same cap is what refuses an over-long voiceline + let Err(err) = process_one_shot(&three_seconds, 1.0) else { + panic!("an over-long one-shot was accepted"); + }; + assert!(err.contains("1 second maximum"), "{}", err); + assert!(process_one_shot(&three_seconds, 5.0).is_ok()); + } +} diff --git a/src/router/custom_song/chart.rs b/src/router/custom_song/chart.rs index c24ff4c..a2831a0 100644 --- a/src/router/custom_song/chart.rs +++ b/src/router/custom_song/chart.rs @@ -1,4 +1,5 @@ use jzon::{object, JsonValue}; +use std::collections::HashMap; // Transcodes a SIF1/NPPS4 beatmap (array of {timing_sec, effect, effect_value, position}) // into the SIF2 chart JSON the client deserializes into NoteData. @@ -87,6 +88,14 @@ fn simultaneous(a: f64, b: f64) -> bool { const MISS_OFFSET_SEC: f64 = 0.15; const MISS_OFFSET_SLIDER_SEC: f64 = 0.34; +// The most notes one difficulty may carry. Every transcode step is linear in this +// and the chart JSON is bounded only in bytes by the upload caps, so an explicit +// ceiling is what keeps a 64MB chart from turning into tens of millions of +// JsonValue allocations on a worker. The hardest shipped SIF2 chart is under 1500 +// notes and the whole 2146-chart official set peaks well below that, so this is +// more than an order of magnitude of headroom +pub const MAX_NOTES: usize = 20_000; + // MarkerData.IsSliderMarker: a chained note with a cross-lane parent or child. fn is_slider(data: &JsonValue, line_of: &dyn Fn(i64) -> Option) -> bool { let parent_id = data["parent_id"].as_i64().unwrap_or(0); @@ -106,10 +115,12 @@ fn is_slider(data: &JsonValue, line_of: &dyn Fn(i64) -> Option) -> bool { // m_MusicDuration is LiveMst._endWait + the music length, and _endWait is 0 in every one of the // 637 official live rows and in ours), so this is what has to fit inside the audio. pub fn end_time(chart: &JsonValue) -> f64 { - let lines: Vec<(i64, i64)> = chart["notes"].members().skip(1) + // Indexed, not scanned: this runs once per note over a chart whose note count + // is only bounded by MAX_NOTES, and the linear lookup made it quadratic + let lines: HashMap = chart["notes"].members().skip(1) .map(|n| (n["id"].as_i64().unwrap_or(0), n["line"].as_i64().unwrap_or(0))) .collect(); - let line_of = |id: i64| lines.iter().find(|(i, _)| *i == id).map(|(_, line)| *line); + let line_of = |id: i64| lines.get(&id).copied(); let mut end: f64 = 0.0; for data in chart["notes"].members().skip(1) { @@ -171,6 +182,12 @@ fn parse_sif_note(data: &JsonValue, index: usize) -> Result<(f64, i64, f64, i64, if !(1..=9).contains(&position) { return Err(format!("Note {}: position {} is outside 1-9", index, position)); } + // NaN and the infinities pass every comparison below and then reach the + // time-order sort, whose partial_cmp().unwrap() panics on an incomparable + // pair - a worker panic authored by the uploaded file + if !timing.is_finite() || !effect_value.is_finite() { + return Err(format!("Note {}: timing_sec/effect_value must be finite numbers", index)); + } if timing < 0.0 { return Err(format!("Note {}: negative timing_sec {}", index, timing)); } @@ -186,17 +203,28 @@ pub fn transcode(beatmap: &JsonValue) -> Result<(JsonValue, i64), String> { if !beatmap.is_array() || beatmap.is_empty() { return Err(String::from("Chart is not a JSON array of notes")); } + if beatmap.len() > MAX_NOTES { + return Err(format!("Chart has {} notes - the maximum is {}", beatmap.len(), MAX_NOTES)); + } let mut work: Vec = Vec::new(); // Slide chain id -> the work indices in that chain, in input order let mut chains: Vec<(i64, Vec)> = Vec::new(); + // (timing bits, position) -> effect, for the duplicate check below. Indexed + // rather than rescanned: the old scan-all-preceding-notes loop was quadratic + // over an input whose size the upload caps only bound in bytes. The key uses + // the raw f64 bits, which is exactly the equality the scan tested (both + // infinities and NaN are already rejected in parse_sif_note, so bit equality + // and value equality agree here) + let mut seen: HashMap<(u64, i64), i64> = HashMap::new(); for (i, data) in beatmap.members().enumerate() { let (timing, effect, effect_value, position, group) = parse_sif_note(data, i)?; - for other in beatmap.members().take(i) { - if other["timing_sec"].as_f64() == Some(timing) && other["position"].as_i64() == Some(position) && other["effect"].as_i64() != Some(effect) { + match seen.insert((timing.to_bits(), position), effect) { + Some(other) if other != effect => { return Err(format!("Note {}: duplicate timing {} on position {} with a different effect", i, timing, position)); - } + }, + _ => {} } let head = work.len(); diff --git a/src/router/custom_song/package.rs b/src/router/custom_song/package.rs index 7a60e69..8efbb9b 100644 --- a/src/router/custom_song/package.rs +++ b/src/router/custom_song/package.rs @@ -41,11 +41,40 @@ pub fn build(music_id: i64) -> Result, String> { Ok(zip.finish().map_err(|e| e.to_string())?.into_inner()) } -fn read_entry(archive: &mut zip::ZipArchive, name: &str) -> Option> { - let mut file = archive.by_name(name).ok()?; +// Reads one entry, capped. Deflate's ceiling is about 1032:1, so an uncapped +// read_to_end here is a zip bomb: a one-megabyte entry inflates to a gigabyte and +// grows the Vec until the allocator or the OOM killer stops it. Every entry is +// bounded by the upload form's own per-file cap, and by what is left of the +// per-request budget across all entries. +// +// The central directory's declared size rejects the obvious case without +// inflating anything; take(cap + 1) makes that declaration untrusted - a lying +// header runs out of budget one byte past the cap and stops there. +// +// Ok(None) is "the package does not carry this entry", which is a normal outcome +// for the optional ones +fn read_entry(archive: &mut zip::ZipArchive, name: &str, remaining: &mut usize) -> Result>, String> { + let Ok(mut file) = archive.by_name(name) else { + return Ok(None); + }; + let cap = std::cmp::min(super::MAX_FILE_BYTES, *remaining); + // Which limit the entry actually ran into, so the message names the right one + let too_big = if cap >= super::MAX_FILE_BYTES { + super::over_file_limit(name) + } else { + super::over_request_limit() + }; + if file.size() > cap as u64 { + return Err(too_big); + } let mut bytes = Vec::new(); - file.read_to_end(&mut bytes).ok()?; - Some(bytes) + file.by_ref().take(cap as u64 + 1).read_to_end(&mut bytes) + .map_err(|_| format!("Package entry '{}' could not be read", name))?; + if bytes.len() > cap { + return Err(too_big); + } + *remaining -= bytes.len(); + Ok(Some(bytes)) } // Expands a package into the same field map the upload form produces - the zip @@ -55,8 +84,10 @@ fn read_entry(archive: &mut zip::ZipArchive, name: &str) -> O // visibility/shared_with/downloads_disabled aren't packaged and stay untouched pub fn expand(package: &[u8], fields: &mut HashMap>) -> Result<(), String> { let mut archive = zip::ZipArchive::new(Cursor::new(package)).map_err(|_| String::from("Package is not a valid zip file"))?; + // The decompressed budget for the whole package, shared by every entry + let mut remaining = super::MAX_REQUEST_BYTES; - let manifest = read_entry(&mut archive, "manifest.json").ok_or(String::from("Package has no manifest.json"))?; + let manifest = read_entry(&mut archive, "manifest.json", &mut remaining)?.ok_or(String::from("Package has no manifest.json"))?; let manifest = jzon::parse(&String::from_utf8_lossy(&manifest)).map_err(|_| String::from("Package manifest is not valid JSON"))?; if manifest["format"].as_i64() != Some(1) { return Err(String::from("Unsupported package format")); @@ -74,11 +105,11 @@ pub fn expand(package: &[u8], fields: &mut HashMap>) -> Result<( } } - fields.insert(String::from("jacket"), read_entry(&mut archive, "jacket").ok_or(String::from("Package has no jacket"))?); - fields.insert(String::from("audio"), read_entry(&mut archive, "audio").ok_or(String::from("Package has no audio"))?); + fields.insert(String::from("jacket"), read_entry(&mut archive, "jacket", &mut remaining)?.ok_or(String::from("Package has no jacket"))?); + fields.insert(String::from("audio"), read_entry(&mut archive, "audio", &mut remaining)?.ok_or(String::from("Package has no audio"))?); let mut has_chart = false; for level in 1..=LEVEL_COUNT { - if let Some(chart) = read_entry(&mut archive, &format!("chart_{}.json", level)) { + if let Some(chart) = read_entry(&mut archive, &format!("chart_{}.json", level), &mut remaining)? { fields.insert(format!("chart_{}", level), chart); has_chart = true; } diff --git a/src/router/lottery.rs b/src/router/lottery.rs index 1a7f159..6c21fe1 100644 --- a/src/router/lottery.rs +++ b/src/router/lottery.rs @@ -427,7 +427,7 @@ async fn lottery_post(req: HttpRequest, Session { key, body }: Session) -> impl } - +// tests!!! #[cfg(test)] mod tests { @@ -447,18 +447,18 @@ mod tests { let mut r1_ids = Vec::new(); for seed in 0..3 { let id = custom_card_db::next_card_id(); - custom_card_db::insert_card(id, 1001, 6001, &object!{ "master_card_id": id, "rarity": 1, "seed": seed }, true, true); + custom_card_db::insert_card(id, 1001, 6001, &object!{ "master_card_id": id, "rarity": 1, "seed": seed }, true, true).unwrap(); r1_ids.push(id); } let r2 = custom_card_db::next_card_id(); - custom_card_db::insert_card(r2, 1001, 6001, &object!{ "master_card_id": r2, "rarity": 2 }, true, true); + custom_card_db::insert_card(r2, 1001, 6001, &object!{ "master_card_id": r2, "rarity": 2 }, true, true).unwrap(); let r3 = custom_card_db::next_card_id(); - custom_card_db::insert_card(r3, 1001, 6001, &object!{ "master_card_id": r3, "rarity": 3 }, true, true); + custom_card_db::insert_card(r3, 1001, 6001, &object!{ "master_card_id": r3, "rarity": 3 }, true, true).unwrap(); // Draft / unobtainable cards must never come out of the pool let draft = custom_card_db::next_card_id(); - custom_card_db::insert_card(draft, 1001, 6001, &object!{ "master_card_id": draft, "rarity": 1 }, false, true); + custom_card_db::insert_card(draft, 1001, 6001, &object!{ "master_card_id": draft, "rarity": 1 }, false, true).unwrap(); let unobtainable = custom_card_db::next_card_id(); - custom_card_db::insert_card(unobtainable, 1001, 6001, &object!{ "master_card_id": unobtainable, "rarity": 1 }, true, false); + custom_card_db::insert_card(unobtainable, 1001, 6001, &object!{ "master_card_id": unobtainable, "rarity": 1 }, true, false).unwrap(); let drawn = custom_banner_cards(11); assert_eq!(drawn.len(), 11); diff --git a/src/router/tools/guest.rs b/src/router/tools/guest.rs index c7da220..41469c5 100644 --- a/src/router/tools/guest.rs +++ b/src/router/tools/guest.rs @@ -71,8 +71,6 @@ pub enum UserView { const DEFAULT_CARD: i64 = 10010001; lazy_static! { - // Each character's lowest official card id: the stand-in shown to viewers - // who can't resolve a custom card of that character static ref OFFICIAL_CARD_BY_CHARACTER: HashMap = { let mut rv: HashMap = HashMap::new(); for entry in databases::CARD_LIST.entries() { @@ -90,8 +88,6 @@ lazy_static! { }; } -// The character behind any card id: baked masterdata first, then the runtime -// custom-card db, then the imported band's id arithmetic (prefix - 9000) fn card_character(id: i64) -> Option { let card = &databases::CARD_LIST[id.to_string()]; if !card.is_empty() { @@ -103,9 +99,6 @@ fn card_character(id: i64) -> Option { Some(id / 10000 - 9000) } -// A custom card the viewer can't resolve shows as its character's base -// official card - the right face, never a crash. Characters with no official -// card (custom ones included) fall back to the default fn proxy_card_id(id: i64) -> i64 { if !card::is_custom(id) { return id; @@ -119,8 +112,6 @@ fn proxy_card_id(id: i64) -> i64 { *rv } -// A card row for a slot the account can't actually supply: level 1, unevolved. -// Only ever handed to viewers, never written back to the account fn stand_in_card(master_card_id: i64) -> JsonValue { object!{ id: master_card_id, @@ -131,13 +122,6 @@ fn stand_in_card(master_card_id: i64) -> JsonValue { } } -// The card object for one of the four favourite/guest slots. global::get_card -// hands back an empty object when the slot is 0 (never set) or names a card the -// account no longer holds, and an empty object reaches the client as -// master_card_id 0: Shock.CardData's constructor looks that up in masterdata and -// dereferences the row, so it throws before the guest cell is ever drawn. Same -// repair userdata::remove_deleted_custom_cards makes for a dead slot - the -// account's first card, then the default for an account holding none fn slot_card(id: i64, user: &JsonValue) -> JsonValue { let card = global::get_card(id, user); if !card.is_empty() { @@ -213,10 +197,6 @@ pub fn get_user(id: i64, friends: &JsonValue, view: UserView, protocol: u32) -> guest_pure_card: slot_card(user["user"]["guest_pure_master_card_id"].as_i64().unwrap_or(0), &user) }; - // The id fields have to name the same card as the objects: surfaces that - // resolve the id instead of the object (profile, friend detail) would - // otherwise look up the slot this just stood in for. A no-op for an account - // whose slots are all set for (key, card) in [ ("favorite_master_card_id", "favorite_card"), ("guest_smile_master_card_id", "guest_smile_card"), @@ -268,6 +248,9 @@ pub fn get_user(id: i64, friends: &JsonValue, view: UserView, protocol: u32) -> rv } + +// here are some tests that ai wrote... + #[cfg(test)] mod tests { use super::*; @@ -326,11 +309,11 @@ mod tests { // A runtime card on an official character proxies to that character let id = db::next_card_id(); - db::insert_card(id, 2003, 5101, &jzon::object!{ "master_card_id": id, "rarity": 1 }, true, false); + db::insert_card(id, 2003, 5101, &jzon::object!{ "master_card_id": id, "rarity": 1 }, true, false).unwrap(); assert_eq!(proxy_card_id(id), 20030001); // On a custom character (no official card) it falls to the default let orphan = db::next_card_id(); - db::insert_card(orphan, db::FIRST_CHARACTER_ID, 5101, &jzon::object!{ "master_card_id": orphan, "rarity": 1 }, true, false); + db::insert_card(orphan, db::FIRST_CHARACTER_ID, 5101, &jzon::object!{ "master_card_id": orphan, "rarity": 1 }, true, false).unwrap(); assert_eq!(proxy_card_id(orphan), DEFAULT_CARD); // A deleted/unknown runtime id can't resolve a character either let unknown = db::next_card_id() + 5000; @@ -353,9 +336,9 @@ mod tests { wipe(5102); let published = db::next_card_id(); - db::insert_card(published, 2003, 5102, &jzon::object!{ "master_card_id": published, "rarity": 1 }, true, false); + db::insert_card(published, 2003, 5102, &jzon::object!{ "master_card_id": published, "rarity": 1 }, true, false).unwrap(); let draft = db::next_card_id(); - db::insert_card(draft, 2003, 5102, &jzon::object!{ "master_card_id": draft, "rarity": 1 }, false, false); + db::insert_card(draft, 2003, 5102, &jzon::object!{ "master_card_id": draft, "rarity": 1 }, false, false).unwrap(); assert!(custom_card::viewer_can_resolve(published, custom_card::PROTOCOL_VERSION)); assert!(!custom_card::viewer_can_resolve(draft, custom_card::PROTOCOL_VERSION)); diff --git a/src/router/userdata/mod.rs b/src/router/userdata/mod.rs index 85ca549..c79a865 100644 --- a/src/router/userdata/mod.rs +++ b/src/router/userdata/mod.rs @@ -97,54 +97,6 @@ INSERT OR IGNORE INTO exchange (user_id, exchange) SELECT user_id, '[]' FROM use } } -// maybe we will use this later -/* -pub fn downgrade_account_cards(user: &JsonValue) -> JsonValue { - let mut rv = user.clone(); - - let mut cards = array![]; - let mut ids = array![]; - for data in user["card_list"].members() { - let id = data["master_card_id"].as_i64().unwrap_or(0); - let downgraded = guest::proxy_card_id(id); - // Whole characters share one downgrade, and the client can't hold the - // same card twice - if ids.contains(downgraded) { - continue; - } - ids.push(downgraded).unwrap(); - let mut data = data.clone(); - if downgraded != id { - data["id"] = downgraded.into(); - data["master_card_id"] = downgraded.into(); - } - cards.push(data).unwrap(); - } - rv["card_list"] = cards; - - for deck in rv["deck_list"].members_mut() { - let mut used = array![]; - for slot in deck["main_card_ids"].members_mut() { - let id = guest::proxy_card_id(slot.as_i64().unwrap_or(0)); - // Cards the downgrade merged away leave the slot empty - if id == 0 || used.contains(id) { - *slot = (0).into(); - continue; - } - used.push(id).unwrap(); - *slot = id.into(); - } - } - - for key in ["favorite_master_card_id", "guest_smile_master_card_id", "guest_cool_master_card_id", "guest_pure_master_card_id"] { - let id = rv["user"][key].as_i64().unwrap_or(0); - rv["user"][key] = guest::proxy_card_id(id).into(); - } - - rv -} -*/ - fn acc_exists(uid: i64) -> bool { DATABASE.lock_and_select("SELECT user_id FROM userdata WHERE user_id=?1", params!(uid)).is_ok() } @@ -247,9 +199,6 @@ fn get_uid(token: &str) -> i64 { data.parse::().unwrap_or(0) } -// The account a login token belongs to, 0 when the token is unknown. The HTTP layer -// never needs this (it keys everything off the token itself), but the multi-live relay -// authenticates a {userId, token} pair and has to check the two agree. pub fn uid_from_login_token(token: &str) -> i64 { get_uid(token) } @@ -301,12 +250,7 @@ fn get_data(auth_key: &str, row: &str) -> JsonValue { jzon::parse(&result.unwrap()).unwrap() } -// Deleted custom songs leave stale score/clear records behind. They're wiped -// lazily when the userdata is pulled: collect the user's own music-id-keyed -// rows in the custom range (official ids are never candidates) and drop the -// ones whose id no longer exists in the catalog. Custom ids are never reused, -// so the wipe is final. A song that still exists but isn't visible to this -// user is NOT wiped - existence is what's checked, not visibility +// Prune deleted custom songs fn remove_deleted_custom_songs(user: &mut JsonValue) -> bool { // Feature off: never touch custom_songs.db, leave userdata untouched if crate::router::custom_song::disabled() { @@ -341,13 +285,7 @@ fn remove_deleted_custom_songs(user: &mut JsonValue) -> bool { true } -// Deleted custom cards leave stale card_list rows behind - and a card_list id -// the client can't resolve aborts its whole login. Wiped lazily when the -// userdata is pulled, mirroring remove_deleted_custom_songs: only the runtime -// band is a candidate (official/imported ids never are), ids are never -// reused, so the wipe is final. A card that still exists but is unpublished -// is NOT wiped - existence is what's checked, and the catalog keeps serving -// owned ids (custom_card::owned_runtime_ids) so holders still resolve them +// Prune deleted custom cards fn remove_deleted_custom_cards(user: &mut JsonValue) -> bool { // Feature off: never touch custom_cards.db, leave userdata untouched if crate::router::custom_card::disabled() { @@ -382,9 +320,6 @@ fn remove_deleted_custom_cards(user: &mut JsonValue) -> bool { } } } - // A dead favorite/guest card repoints to the account's first remaining - // card (every account has its tutorial cards; the fallback can't trigger - // in practice) let fallback = user["card_list"][0]["master_card_id"].as_i64().unwrap_or(10010001); for key in ["favorite_master_card_id", "guest_smile_master_card_id", "guest_cool_master_card_id", "guest_pure_master_card_id"] { if dead.contains(user["user"][key].as_i64().unwrap_or(0)) { @@ -484,22 +419,6 @@ pub fn save_server_data(auth_key: &str, data: JsonValue) { save_data(auth_key, "server_data", data); } -// Read-modify-write of one account's server_data as ONE atomic step. -// -// get_server_data + save_server_data open a connection each, so two requests for the same -// account both read the pre-state and the later write wins - which for a record that is -// meant to be spent exactly once (the started-live record /multi_live/end awards off) means -// both ends see it and both award. Everything here happens inside a single BEGIN IMMEDIATE -// transaction instead, so concurrent callers serialise and the second one observes what the -// first one wrote. -// -// `f` sees the parsed server_data and returns whatever the caller needs out of it; the -// (possibly mutated) value is written back before the transaction commits. get_key runs -// BEFORE the transaction because it can create the account, which writes on its own -// connection and would otherwise deadlock against our write lock. -// -// A database error yields T::default() rather than a panic: the callers all have a -// "nothing to spend" branch, which is the right answer when the record could not be read. pub fn modify_server_data(auth_key: &str, f: impl FnOnce(&mut JsonValue) -> T) -> T { let key = get_key(auth_key); let rv = DATABASE.lock_and_transact(|conn| { @@ -788,15 +707,6 @@ pub fn export_user(token: &str) -> Option { }) } -// Every row an account owns, gone. Factored out of purge_accounts so the arcade -// sweeper (a machine that aged out takes its two accounts with it) and the -// card-rebind orphan cleanup delete exactly the same set of rows - an account -// half-deleted here is one the login path would resurrect empty. -// -// This list is also what the arcade guest reset mirrors: starter::write_starter_rows -// rewrites the eleven data rows, re-draws the token and deletes the rest, so a -// row added here has to be accounted for there too or a guest starts carrying -// the previous player's state across a credit. pub const ACCOUNT_TABLES: &[&str] = &[ "userdata", "userhome", "missions", "loginbonus", "sifcards", "friends", "chats", "exchange", "event", "eventloginbonus", "server_data", "webui", @@ -808,11 +718,12 @@ pub fn delete_account(user_id: i64) { for table in ACCOUNT_TABLES { DATABASE.lock_and_exec(&format!("DELETE FROM {} WHERE user_id=?1", table), params!(user_id)); } + + crate::router::custom_song::purge_owner(user_id); + crate::router::custom_card::purge_owner(user_id); + crate::router::custom_3dmv::purge_owner(user_id); } -// True when the account has ever registered a data-transfer password, which is -// the only way an account can be taken over from another device. The arcade uses -// it to tell a throwaway account it made itself from a real player's account. pub fn has_transfer_password(user_id: i64) -> bool { !DATABASE.lock_and_select("SELECT password FROM migration WHERE user_id=?1", params!(user_id)) .unwrap_or_default() @@ -841,6 +752,9 @@ pub fn purge_accounts() -> usize { dead_uids.len() } + +// more tests??? + #[cfg(test)] mod tests { use super::*; @@ -855,10 +769,10 @@ mod tests { let mut user = get_acc(token); let deleted_id = custom_song::next_music_id(); - custom_song::insert_song(deleted_id, 1, &object!{music_id: deleted_id}, "public", &array![], false); + custom_song::insert_song(deleted_id, 1, &object!{music_id: deleted_id}, "public", &array![], false).unwrap(); // Exists but isn't visible to this user - must survive the wipe let private_id = custom_song::next_music_id(); - custom_song::insert_song(private_id, 1, &object!{music_id: private_id}, "private", &array![], false); + custom_song::insert_song(private_id, 1, &object!{music_id: private_id}, "private", &array![], false).unwrap(); // 1100101 is a real stock live-id shape (7-digit, >= FIRST_MUSIC_ID): it // must survive the custom-song wipe, unlike an unrealistic sub-10000 id @@ -883,7 +797,7 @@ mod tests { assert_eq!(user["live_list"].len(), 3); assert_eq!(user["live_mission_list"].len(), 3); - custom_song::delete_song(deleted_id); + custom_song::delete_song(deleted_id).unwrap(); // The next pull drops the dead id's records and only those let user = get_acc(token); @@ -912,10 +826,10 @@ mod tests { let mut user = get_acc(token); let deleted_id = custom_card::next_card_id(); - custom_card::insert_card(deleted_id, 1001, 1, &jzon::object!{ "master_card_id": deleted_id, "rarity": 1 }, true, true); + custom_card::insert_card(deleted_id, 1001, 1, &jzon::object!{ "master_card_id": deleted_id, "rarity": 1 }, true, true).unwrap(); // Exists but was unpublished - must survive the wipe let unpublished_id = custom_card::next_card_id(); - custom_card::insert_card(unpublished_id, 1001, 1, &jzon::object!{ "master_card_id": unpublished_id, "rarity": 1 }, false, false); + custom_card::insert_card(unpublished_id, 1001, 1, &jzon::object!{ "master_card_id": unpublished_id, "rarity": 1 }, false, false).unwrap(); for id in [deleted_id, unpublished_id] { user["card_list"].push(jzon::object!{ diff --git a/src/router/webui.rs b/src/router/webui.rs index c94539d..807188e 100644 --- a/src/router/webui.rs +++ b/src/router/webui.rs @@ -25,10 +25,13 @@ fn get_config() -> JsonValue { pub fn get_login_token(req: &HttpRequest) -> Option { let blank_header = HeaderValue::from_static(""); let cookies = req.headers().get("Cookie").unwrap_or(&blank_header).to_str().unwrap_or(""); - if cookies.is_empty() { - return None; + for pair in cookies.split(';') { + let Some((name, value)) = pair.split_once('=') else { continue; }; + if name.trim() == "ew_token" { + return Some(value.trim().to_string()); + } } - Some(cookies.split("ew_token=").last().unwrap_or("").split(';').collect::>()[0].to_string()) + None } fn session_uid(req: &HttpRequest) -> Option { @@ -607,13 +610,6 @@ pub fn remove_arcade_machine(req: HttpRequest, body: String) -> HttpResponse { .body(jzon::stringify(resp)) } -// The account page's "bind arcade card" form: the card id, and nothing else. -// The webui session already proves whose account this is, so the card is bound -// to the signed-in account through arcade::bind_card_to - the same rule the -// cabinet's /api/arcade/bind applies once its own proof, the transfer code and -// password, has named the account. The cabinet endpoint speaks the encrypted -// game protocol behind the asset gate, which a browser cannot, so this is the -// browser's door onto that rule rather than a copy of it. pub fn bind_arcade_card(req: HttpRequest, body: String) -> HttpResponse { if crate::router::arcade::disabled() { return HttpResponse::NotFound().finish(); @@ -687,6 +683,28 @@ pub fn cheat(req: HttpRequest, _body: String) -> HttpResponse { mod tests { use super::*; + // get_login_token is the ONLY authentication on every mutating custom-content + // route, so it has to read the cookie header as cookies, not as a substring: + // a name that merely ends in ew_token, or a second ew_token= appended later, + // must not win over the real session cookie + #[test] + fn the_session_cookie_is_matched_by_name() { + let token_for = |header: &str| get_login_token( + &actix_web::test::TestRequest::default().insert_header(("Cookie", header)).to_http_request() + ); + + assert_eq!(token_for("ew_token=real").as_deref(), Some("real")); + assert_eq!(token_for("theme=dark; ew_token=real; lang=en").as_deref(), Some("real")); + // A cookie whose NAME ends in ew_token is a different cookie + assert_eq!(token_for("not_ew_token=attacker").as_deref(), None); + assert_eq!(token_for("ew_token=real; xew_token=attacker").as_deref(), Some("real")); + // A duplicate set later in the header does not override the first + assert_eq!(token_for("ew_token=real; ew_token=attacker").as_deref(), Some("real")); + // No cookie at all is no session, not the whole header + assert_eq!(token_for("theme=dark").as_deref(), None); + assert_eq!(get_login_token(&actix_web::test::TestRequest::default().to_http_request()), None); + } + // The picker lists the card form searches by name: every baked character // (official + SIF1 import, badged apart) and every skill_center row with // its JP and EN display strings diff --git a/src/sql.rs b/src/sql.rs index b199c28..7ee2feb 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -4,14 +4,20 @@ use jzon::{JsonValue, array}; pub struct SQLite { path: String } +const BUSY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + +fn open(path: &str) -> Result { + let conn = Connection::open(path)?; + conn.busy_timeout(BUSY_TIMEOUT)?; + Ok(conn) +} impl SQLite { pub fn new(path: &str, setup: fn(&Connection)) -> SQLite { let instance = SQLite { path: crate::get_data_path(path) }; - let conn = Connection::open(&instance.path).unwrap(); - conn.busy_timeout(std::time::Duration::from_secs(10)).unwrap(); + let conn = open(&instance.path).unwrap(); conn.execute("PRAGMA foreign_keys = ON;", ()).unwrap(); setup(&conn); instance @@ -20,11 +26,11 @@ impl SQLite { &self.path } pub fn lock_and_exec(&self, command: &str, args: &[&dyn ToSql]) { - let conn = Connection::open(&self.path).unwrap(); + let conn = open(&self.path).unwrap(); conn.execute(command, args).unwrap(); } pub fn lock_and_select(&self, command: &str, args: &[&dyn ToSql]) -> Result { - let conn = Connection::open(&self.path).unwrap(); + let conn = open(&self.path)?; let mut stmt = conn.prepare(command)?; stmt.query_row(args, |row| { match row.get::(0) { @@ -34,14 +40,14 @@ impl SQLite { }) } pub fn lock_and_select_type(&self, command: &str, args: &[&dyn ToSql]) -> Result { - let conn = Connection::open(&self.path).unwrap(); + let conn = open(&self.path)?; let mut stmt = conn.prepare(command)?; stmt.query_row(args, |row| { row.get(0) }) } pub fn lock_and_select_all(&self, command: &str, args: &[&dyn ToSql]) -> Result { - let conn = Connection::open(&self.path).unwrap(); + let conn = open(&self.path)?; let mut stmt = conn.prepare(command)?; let map = stmt.query_map(args, |row| { match row.get::(0) { @@ -60,24 +66,11 @@ impl SQLite { Ok(rv) } - // Runs a read-modify-write as one unit. Additive on purpose — the other helpers open - // a fresh connection per statement, so a caller that SELECTs then INSERTs races any - // concurrent caller doing the same and the loser hits a constraint violation (which - // lock_and_exec would unwrap into a worker panic). - // - // BEGIN IMMEDIATE takes the write lock up front rather than at first write, so two - // callers serialise instead of both reading the pre-state; busy_timeout makes the - // loser wait for the winner rather than fail instantly (SQLite::new sets that on its - // own short-lived setup connection, not on the per-call ones). - // - // Errors are returned, never unwrapped: statistics writes must not take down a - // request. pub fn lock_and_transact( &self, f: impl FnOnce(&Connection) -> Result ) -> Result { - let mut conn = Connection::open(&self.path)?; - conn.busy_timeout(std::time::Duration::from_secs(10))?; + let mut conn = open(&self.path)?; let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; let rv = f(&tx)?; tx.commit()?;