4 Commits
Author SHA1 Message Date
Ethan O'Brien b92acb7a00 Mac -> macOS 2026-08-28 22:32:16 -05:00
Ethan O'Brien 971aa32361 Add macos route and some other tests I guess 2026-08-28 21:40:26 -05:00
Ethan O'Brien 2ceff8ffc6 commit new webui 2026-08-28 19:45:37 -05:00
Ethan O'Brien ea39c12a9e Some design fixes in custom data handling 2026-08-28 19:43:56 -05:00
19 changed files with 2141 additions and 343 deletions
+58 -12
View File
@@ -53,12 +53,18 @@ pub fn next_mv_id() -> i64 {
std::cmp::max(std::cmp::max(issued, max), FIRST_MV_ID - 1) + 1
}
pub fn insert_mv(mv_id: i64, music_id: i64, owner_id: i64, mv: &JsonValue, published: bool) {
DATABASE.lock_and_exec(
// The row and the high-water mark are one write: a failure between them leaves an
// MV whose id the next upload would reissue, and the failure has to reach the
// uploader as an error rather than panic the worker (lock_and_exec unwraps)
pub fn insert_mv(mv_id: i64, music_id: i64, owner_id: i64, mv: &JsonValue, published: bool) -> Result<(), rusqlite::Error> {
DATABASE.lock_and_transact(|conn| {
conn.execute(
"INSERT INTO mvs (mv_id, music_id, owner_id, mv, published) VALUES (?1, ?2, ?3, ?4, ?5)",
params!(mv_id, music_id, owner_id, jzon::stringify(mv.clone()), published as i64)
);
DATABASE.lock_and_exec("INSERT INTO revision (id, revision, last_mv_id) VALUES (1, 0, ?1) ON CONFLICT(id) DO UPDATE SET last_mv_id=?1", params!(mv_id));
)?;
conn.execute("INSERT INTO revision (id, revision, last_mv_id) VALUES (1, 0, ?1) ON CONFLICT(id) DO UPDATE SET last_mv_id=?1", params!(mv_id))?;
Ok(())
})
}
// The catalog blob only. The owner and the published flag live in their own
@@ -195,17 +201,24 @@ pub fn all_mv_blobs() -> Option<JsonValue> {
// Blobs are content-addressed and may be shared between MVs (or roles), so
// every candidate row is checked - a single-row scan could land on a
// coincidental substring match and miss the real reference
pub fn blob_in_use(md5: &str) -> bool {
let rows = DATABASE.lock_and_select_all("SELECT mv FROM mvs WHERE mv LIKE ?1", params!(format!("%{}%", md5))).unwrap_or(array![]);
// coincidental substring match and miss the real reference.
//
// Returns Result and never folds an error into "not referenced": the caller
// unlinks on false, and a read that failed (SQLITE_BUSY under a concurrent
// write, a corrupt page) says nothing about whether the blob is live. The
// startup sweep is deliberately fail-closed; this is its online half
pub fn blob_in_use(md5: &str) -> Result<bool, rusqlite::Error> {
let rows = DATABASE.lock_and_select_all("SELECT mv FROM mvs WHERE mv LIKE ?1", params!(format!("%{}%", md5)))?;
for blob in rows.members() {
if let Ok(mv) = jzon::parse(&blob.to_string()) {
let Ok(mv) = jzon::parse(&blob.to_string()) else {
// An unparseable row could reference anything - assume it does
return Ok(true);
};
if mv["files"].members().any(|file| file["md5"].as_str() == Some(md5)) {
return true;
return Ok(true);
}
}
}
false
Ok(false)
}
// Two-step content-addressed lookup for the data route: the LIKE scan finds a
@@ -213,5 +226,38 @@ pub fn blob_in_use(md5: &str) -> bool {
// stored file of a live MV (and not a substring coincidence elsewhere in the
// blob). The blob path itself derives from the md5
pub fn find_blob_by_md5(md5: &str) -> bool {
blob_in_use(md5)
// A read failure here means "cannot prove this blob is served", which the
// route turns into a 404 - the client re-requests. Unlike the GC, guessing
// wrong in this direction costs nothing permanent
blob_in_use(md5).unwrap_or(false)
}
// Every MV this account uploaded, for the account-deletion purge
pub fn mv_ids_for_owner(owner_id: i64) -> Vec<i64> {
let rows = DATABASE.lock_and_select_all("SELECT mv_id FROM mvs WHERE owner_id=?1 ORDER BY mv_id", params!(owner_id)).unwrap_or(array![]);
rows.members().filter_map(|id| id.as_i64()).collect()
}
// The stored bytes an MV's files account for. Blobs are shared, so two MVs that
// carry the same model both count it: the quota is "what this account asked the
// server to store", not a dedup-aware disk figure
pub fn mv_bytes(files: &JsonValue) -> i64 {
files.members().map(|file| file["size"].as_i64().unwrap_or(0)).sum()
}
// What this account's MVs already occupy, optionally ignoring one (the MV being
// replaced by an in-place edit, whose new size is counted instead)
pub fn owner_bytes(owner_id: i64, excluding_mv_id: i64) -> i64 {
let rows = parse_blobs(DATABASE.lock_and_select_all("SELECT mv FROM mvs WHERE owner_id=?1", params!(owner_id)).unwrap_or(array![]));
rows.members()
.filter(|mv| mv["mv_id"].as_i64() != Some(excluding_mv_id))
.map(|mv| mv_bytes(&mv["files"]))
.sum()
}
// The on-disk database file, so a test can force the read errors the fail-closed
// GC paths are built for
#[cfg(test)]
pub fn test_db_path() -> String {
DATABASE.get_path().to_string()
}
+109 -27
View File
@@ -87,20 +87,29 @@ pub fn next_character_id() -> i64 {
std::cmp::max(std::cmp::max(issued, max), FIRST_CHARACTER_ID - 1) + 1
}
pub fn insert_card(master_card_id: i64, master_character_id: i64, owner_id: i64, card: &JsonValue, published: bool, obtainable: bool) {
DATABASE.lock_and_exec(
// The row and the high-water mark are one write: a failure between them leaves a
// card whose id the next upload would reissue, and the failure has to reach the
// uploader as an error rather than panic the worker (lock_and_exec unwraps)
pub fn insert_card(master_card_id: i64, master_character_id: i64, owner_id: i64, card: &JsonValue, published: bool, obtainable: bool) -> Result<(), rusqlite::Error> {
DATABASE.lock_and_transact(|conn| {
conn.execute(
"INSERT INTO cards (master_card_id, master_character_id, owner_id, card, rarity, published, obtainable) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
params!(master_card_id, master_character_id, owner_id, jzon::stringify(card.clone()), card["rarity"].as_i64().unwrap_or(1), published as i64, obtainable as i64)
);
DATABASE.lock_and_exec("INSERT INTO revision (id, revision, last_card_id, last_character_id) VALUES (1, 0, ?1, 0) ON CONFLICT(id) DO UPDATE SET last_card_id=?1", params!(master_card_id));
)?;
conn.execute("INSERT INTO revision (id, revision, last_card_id, last_character_id) VALUES (1, 0, ?1, 0) ON CONFLICT(id) DO UPDATE SET last_card_id=?1", params!(master_card_id))?;
Ok(())
})
}
pub fn insert_character(master_character_id: i64, owner_id: i64, character: &JsonValue) {
DATABASE.lock_and_exec(
pub fn insert_character(master_character_id: i64, owner_id: i64, character: &JsonValue) -> Result<(), rusqlite::Error> {
DATABASE.lock_and_transact(|conn| {
conn.execute(
"INSERT INTO characters (master_character_id, owner_id, character) VALUES (?1, ?2, ?3)",
params!(master_character_id, owner_id, jzon::stringify(character.clone()))
);
DATABASE.lock_and_exec("INSERT INTO revision (id, revision, last_card_id, last_character_id) VALUES (1, 0, 0, ?1) ON CONFLICT(id) DO UPDATE SET last_character_id=?1", params!(master_character_id));
)?;
conn.execute("INSERT INTO revision (id, revision, last_card_id, last_character_id) VALUES (1, 0, 0, ?1) ON CONFLICT(id) DO UPDATE SET last_character_id=?1", params!(master_character_id))?;
Ok(())
})
}
// The catalog blob only. The owner and the published/obtainable flags live in
@@ -173,6 +182,47 @@ pub fn has_character(master_character_id: i64) -> bool {
DATABASE.lock_and_select("SELECT master_character_id FROM characters WHERE master_character_id=?1", params!(master_character_id)).is_ok()
}
// Every card / character this account uploaded, for the account-deletion purge
pub fn card_ids_for_owner(owner_id: i64) -> Vec<i64> {
DATABASE.lock_and_select_all("SELECT master_card_id FROM cards WHERE owner_id=?1 ORDER BY master_card_id", params!(owner_id))
.unwrap_or(array![]).members().filter_map(|id| id.as_i64()).collect()
}
pub fn character_ids_for_owner(owner_id: i64) -> Vec<i64> {
DATABASE.lock_and_select_all("SELECT master_character_id FROM characters WHERE owner_id=?1 ORDER BY master_character_id", params!(owner_id))
.unwrap_or(array![]).members().filter_map(|id| id.as_i64()).collect()
}
// The stored bytes one card / character accounts for: every derived art png
// plus, for a character, its voiceline oggs
pub fn card_bytes(card: &JsonValue) -> i64 {
card["art"].members().map(|art| art["size"].as_i64().unwrap_or(0)).sum()
}
pub fn character_bytes(character: &JsonValue) -> i64 {
let art: i64 = character["art"].members().map(|art| art["size"].as_i64().unwrap_or(0)).sum();
let voice: i64 = character["voice"].members().map(|line| line["size"].as_i64().unwrap_or(0)).sum();
art + voice
}
// What this account's uploads already occupy. Cards and characters share one
// quota (they share one storage root), each optionally ignoring the entity being
// replaced by an in-place edit, whose new size is counted instead
pub fn owner_bytes(owner_id: i64, excluding_card_id: i64, excluding_character_id: i64) -> i64 {
let mut total = 0;
for blob in parse_blobs(DATABASE.lock_and_select_all("SELECT card FROM cards WHERE owner_id=?1", params!(owner_id)).unwrap_or(array![])).members() {
if blob["master_card_id"].as_i64() != Some(excluding_card_id) {
total += card_bytes(blob);
}
}
for blob in parse_blobs(DATABASE.lock_and_select_all("SELECT character FROM characters WHERE owner_id=?1", params!(owner_id)).unwrap_or(array![])).members() {
if blob["master_character_id"].as_i64() != Some(excluding_character_id) {
total += character_bytes(blob);
}
}
total
}
pub fn card_count_for_owner(owner_id: i64) -> i64 {
DATABASE.lock_and_select_type::<i64>("SELECT COUNT(*) FROM cards WHERE owner_id=?1", params!(owner_id)).unwrap_or(0)
}
@@ -300,7 +350,10 @@ pub fn get_browse_cards() -> JsonValue {
// considered, and ids are never reused, so official (or imported) cards can't
// come back from this and a wipe is final. A card that's merely unpublished
// still has its row - only genuinely deleted ids are returned
pub fn dead_card_ids(candidates: &JsonValue) -> JsonValue {
// None = the catalog could not be read (or is entirely empty while players still hold
// custom ids): the caller must prune NOTHING then. An unreadable catalog looks exactly
// like one where every id is dead, and get_acc saves the pruned userdata.
pub fn dead_card_ids(candidates: &JsonValue) -> Option<JsonValue> {
let mut ids: Vec<i64> = Vec::new();
for id in candidates.members() {
let Some(id) = id.as_i64() else { continue; };
@@ -309,17 +362,23 @@ pub fn dead_card_ids(candidates: &JsonValue) -> JsonValue {
}
}
if ids.is_empty() {
return array![];
return Some(array![]);
}
let list = ids.iter().map(|id| id.to_string()).collect::<Vec<_>>().join(",");
let alive = DATABASE.lock_and_select_all(&format!("SELECT master_card_id FROM cards WHERE master_card_id IN ({})", list), params!()).unwrap_or(array![]);
let alive = DATABASE.lock_and_select_all(&format!("SELECT master_card_id FROM cards WHERE master_card_id IN ({})", list), params!()).ok()?;
if alive.is_empty() {
let total: i64 = DATABASE.lock_and_select_type("SELECT COUNT(*) FROM cards", params!()).ok()?;
if total == 0 {
return None;
}
}
let mut rv = array![];
for id in ids {
if !alive.contains(id) {
rv.push(id).unwrap();
}
}
rv
Some(rv)
}
// The published + obtainable pool the custom gacha banner draws from, per
@@ -410,10 +469,10 @@ mod tests {
let first = next_card_id();
assert!(first >= FIRST_CARD_ID);
assert_ne!(first % 10000, 0);
insert_card(first, 1001, 3001, &card_blob(first, 1), false, false);
insert_card(first, 1001, 3001, &card_blob(first, 1), false, false).unwrap();
let second = next_card_id();
assert_eq!(second, first + 1);
insert_card(second, 1001, 3001, &card_blob(second, 1), false, false);
insert_card(second, 1001, 3001, &card_blob(second, 1), false, false).unwrap();
delete_card(second);
assert_eq!(get_card(second), None);
// The high-water mark survives the delete, so the id is retired
@@ -421,7 +480,7 @@ mod tests {
let character = next_character_id();
assert!(character >= FIRST_CHARACTER_ID);
insert_character(character, 3001, &object!{ "master_character_id": character });
insert_character(character, 3001, &object!{ "master_character_id": character }).unwrap();
assert_eq!(next_character_id(), character + 1);
delete_character(character);
assert_eq!(next_character_id(), character + 1);
@@ -440,13 +499,13 @@ mod tests {
wipe(3004);
let character = next_character_id();
insert_character(character, 3003, &object!{ "master_character_id": character });
insert_character(character, 3003, &object!{ "master_character_id": character }).unwrap();
let published = next_card_id();
let mut blob = card_blob(published, 3);
blob["master_character_id"] = character.into();
insert_card(published, character, 3003, &blob, true, true);
insert_card(published, character, 3003, &blob, true, true).unwrap();
let draft = next_card_id();
insert_card(draft, character, 3003, &card_blob(draft, 1), false, false);
insert_card(draft, character, 3003, &card_blob(draft, 1), false, false).unwrap();
let owner_view = get_cards_for_user(3003, &[]);
assert_eq!(owner_view.len(), 2);
@@ -493,13 +552,13 @@ mod tests {
wipe(3005);
let r1 = next_card_id();
insert_card(r1, 1001, 3005, &card_blob(r1, 1), true, true);
insert_card(r1, 1001, 3005, &card_blob(r1, 1), true, true).unwrap();
let r3 = next_card_id();
insert_card(r3, 1001, 3005, &card_blob(r3, 3), true, true);
insert_card(r3, 1001, 3005, &card_blob(r3, 3), true, true).unwrap();
let unpublished = next_card_id();
insert_card(unpublished, 1001, 3005, &card_blob(unpublished, 1), false, true);
insert_card(unpublished, 1001, 3005, &card_blob(unpublished, 1), false, true).unwrap();
let unobtainable = next_card_id();
insert_card(unobtainable, 1001, 3005, &card_blob(unobtainable, 1), true, false);
insert_card(unobtainable, 1001, 3005, &card_blob(unobtainable, 1), true, false).unwrap();
assert_eq!(obtainable_card_ids(1), vec![r1]);
assert_eq!(obtainable_card_ids(3), vec![r3]);
@@ -516,7 +575,7 @@ mod tests {
wipe(3007);
let id = next_card_id();
insert_card(id, 1001, 3007, &card_blob(id, 1), true, false);
insert_card(id, 1001, 3007, &card_blob(id, 1), true, false).unwrap();
assert_eq!(find_asset_by_md5(&format!("{:032x}", id)), Some(format!("{}/c_00.png", id)));
assert_eq!(find_asset_by_md5("00000000000000000000000000000000"), None);
@@ -524,7 +583,7 @@ mod tests {
insert_character(character, 3007, &object!{
"master_character_id": character,
"art": [{ "kind": "icon", "md5": "aabbccddeeff00112233445566778899", "size": 1 }]
});
}).unwrap();
assert_eq!(
find_asset_by_md5("aabbccddeeff00112233445566778899"),
Some(format!("characters/{}/icon.png", character))
@@ -541,15 +600,38 @@ mod tests {
wipe(3008);
let alive = next_card_id();
insert_card(alive, 1001, 3008, &card_blob(alive, 1), false, false);
insert_card(alive, 1001, 3008, &card_blob(alive, 1), false, false).unwrap();
let dead = next_card_id();
insert_card(dead, 1001, 3008, &card_blob(dead, 1), true, false);
insert_card(dead, 1001, 3008, &card_blob(dead, 1), true, false).unwrap();
delete_card(dead);
let dead_ids = dead_card_ids(&array![alive, dead, 10010001, 100010001, dead]);
let dead_ids = dead_card_ids(&array![alive, dead, 10010001, 100010001, dead]).unwrap();
assert_eq!(dead_ids.len(), 1);
assert_eq!(dead_ids[0].as_i64(), Some(dead));
wipe(3008);
}
// An unreadable or blank catalog must never read as "every custom card is
// dead": get_acc prunes on that answer and saves the pruned userdata
#[test]
fn dead_ids_are_unknown_when_the_catalog_cannot_be_read() {
let _lock = crate::runtime::lock_test_data_path();
wipe(3009);
let alive = next_card_id();
insert_card(alive, 1001, 3009, &card_blob(alive, 1), false, false).unwrap();
let conn = rusqlite::Connection::open(DATABASE.get_path()).unwrap();
conn.execute("ALTER TABLE cards RENAME TO cards_hidden", ()).unwrap();
let unreadable = dead_card_ids(&array![alive]);
conn.execute("CREATE TABLE cards (master_card_id BIGINT NOT NULL PRIMARY KEY, master_character_id BIGINT NOT NULL, owner_id BIGINT NOT NULL, card TEXT NOT NULL, rarity INT NOT NULL DEFAULT 1, published INT NOT NULL DEFAULT 0, obtainable INT NOT NULL DEFAULT 0)", ()).unwrap();
let blank = dead_card_ids(&array![alive]);
conn.execute("DROP TABLE cards", ()).unwrap();
conn.execute("ALTER TABLE cards_hidden RENAME TO cards", ()).unwrap();
assert!(unreadable.is_none(), "an unreadable catalog reported dead cards");
assert!(blank.is_none(), "a blank catalog reported every card dead");
assert_eq!(dead_card_ids(&array![alive]).unwrap().len(), 0);
wipe(3009);
}
}
+135 -20
View File
@@ -107,10 +107,16 @@ pub fn next_music_id() -> i64 {
std::cmp::max(std::cmp::max(issued, max), FIRST_MUSIC_ID - 1) + 1
}
pub fn insert_song(music_id: i64, owner_id: i64, song: &JsonValue, visibility: &str, shared_with: &JsonValue, downloads_disabled: bool) {
DATABASE.lock_and_exec("INSERT INTO songs (music_id, owner_id, song, visibility, downloads_disabled) VALUES (?1, ?2, ?3, ?4, ?5)", params!(music_id, owner_id, jzon::stringify(song.clone()), visibility, downloads_disabled as i64));
DATABASE.lock_and_exec("INSERT INTO revision (id, revision, last_music_id) VALUES (1, 0, ?1) ON CONFLICT(id) DO UPDATE SET last_music_id=?1", params!(music_id));
set_shared_users(music_id, shared_with);
// The row, the high-water mark and the shared-user list are one write: a failure
// between them would leave a song whose id the next upload reissues, and the
// failure itself must reach the uploader as an error rather than panic the worker
// (lock_and_exec unwraps, lock_and_transact returns)
pub fn insert_song(music_id: i64, owner_id: i64, song: &JsonValue, visibility: &str, shared_with: &JsonValue, downloads_disabled: bool) -> Result<(), rusqlite::Error> {
DATABASE.lock_and_transact(|conn| {
conn.execute("INSERT INTO songs (music_id, owner_id, song, visibility, downloads_disabled) VALUES (?1, ?2, ?3, ?4, ?5)", params!(music_id, owner_id, jzon::stringify(song.clone()), visibility, downloads_disabled as i64))?;
conn.execute("INSERT INTO revision (id, revision, last_music_id) VALUES (1, 0, ?1) ON CONFLICT(id) DO UPDATE SET last_music_id=?1", params!(music_id))?;
write_shared_users(conn, music_id, shared_with)
})
}
// Replace an existing song's catalog blob in place. The owner, visibility,
@@ -119,9 +125,12 @@ pub fn update_song(music_id: i64, song: &JsonValue) {
DATABASE.lock_and_exec("UPDATE songs SET song=?1 WHERE music_id=?2", params!(jzon::stringify(song.clone()), music_id));
}
pub fn delete_song(music_id: i64) {
DATABASE.lock_and_exec("DELETE FROM songs WHERE music_id=?1", params!(music_id));
DATABASE.lock_and_exec("DELETE FROM shared_users WHERE music_id=?1", params!(music_id));
pub fn delete_song(music_id: i64) -> Result<(), rusqlite::Error> {
DATABASE.lock_and_transact(|conn| {
conn.execute("DELETE FROM songs WHERE music_id=?1", params!(music_id))?;
conn.execute("DELETE FROM shared_users WHERE music_id=?1", params!(music_id))?;
Ok(())
})
}
pub fn get_song(music_id: i64) -> Option<JsonValue> {
@@ -133,9 +142,11 @@ pub fn get_song_owner(music_id: i64) -> Option<i64> {
DATABASE.lock_and_select("SELECT owner_id FROM songs WHERE music_id=?1", params!(music_id)).ok()?.parse::<i64>().ok()
}
pub fn set_visibility(music_id: i64, visibility: &str, shared_with: &JsonValue) {
DATABASE.lock_and_exec("UPDATE songs SET visibility=?1 WHERE music_id=?2", params!(visibility, music_id));
set_shared_users(music_id, shared_with);
pub fn set_visibility(music_id: i64, visibility: &str, shared_with: &JsonValue) -> Result<(), rusqlite::Error> {
DATABASE.lock_and_transact(|conn| {
conn.execute("UPDATE songs SET visibility=?1 WHERE music_id=?2", params!(visibility, music_id))?;
write_shared_users(conn, music_id, shared_with)
})
}
pub fn set_downloads_disabled(music_id: i64, downloads_disabled: bool) {
@@ -146,11 +157,15 @@ fn get_downloads_disabled(music_id: i64) -> bool {
DATABASE.lock_and_select("SELECT downloads_disabled FROM songs WHERE music_id=?1", params!(music_id)).unwrap_or_default() == "1"
}
fn set_shared_users(music_id: i64, shared_with: &JsonValue) {
DATABASE.lock_and_exec("DELETE FROM shared_users WHERE music_id=?1", params!(music_id));
// Replace-the-whole-list, inside the caller's transaction: a half-written list is a
// song shared with the wrong people
fn write_shared_users(conn: &rusqlite::Connection, music_id: i64, shared_with: &JsonValue) -> Result<(), rusqlite::Error> {
conn.execute("DELETE FROM shared_users WHERE music_id=?1", params!(music_id))?;
for id in shared_with.members() {
DATABASE.lock_and_exec("INSERT OR IGNORE INTO shared_users (music_id, user_id) VALUES (?1, ?2)", params!(music_id, id.as_i64().unwrap()));
let Some(id) = id.as_i64() else { continue; };
conn.execute("INSERT OR IGNORE INTO shared_users (music_id, user_id) VALUES (?1, ?2)", params!(music_id, id))?;
}
Ok(())
}
fn get_visibility(music_id: i64) -> String {
@@ -257,6 +272,67 @@ pub fn public_song_title(music_id: i64, english: bool) -> Option<String> {
Some(if english && !name_en.is_empty() { name_en } else { name })
}
// Whether this viewer may fetch the song's per-id asset files (jacket, blur,
// chart JSON). Exactly the catalog's visibility rule: public to everyone, private
// to its owner, shared to its owner and the shared-user list. Anonymous viewers
// (no webui session) are only ever shown public songs.
//
// The /data/{md5} route stays sessionless on purpose - a 128-bit content hash IS
// the capability there - but /assets/{music_id}/{file} is addressed by a
// sequential id, so it needs the real rule
pub fn asset_visible(music_id: i64, viewer: Option<i64>) -> bool {
let Some(owner) = get_song_owner(music_id) else {
return false;
};
let viewer = viewer.unwrap_or(0);
if owner == viewer {
return true;
}
match get_visibility(music_id).as_str() {
"public" => true,
"shared" => get_shared_users(music_id).contains(viewer),
_ => false
}
}
// Every song this account uploaded, for the account-deletion purge
pub fn music_ids_by_owner(owner_id: i64) -> Vec<i64> {
DATABASE.lock_and_select_all("SELECT music_id FROM songs WHERE owner_id=?1 ORDER BY music_id", params!(owner_id))
.unwrap_or(array![])
.members().filter_map(|id| id.as_i64()).collect()
}
// The served bytes one song accounts for: both jackets, every chart and both
// audio cues, straight out of the catalog blob that quotes them to the client.
// The original upload artifacts kept under original/ are NOT counted (the
// catalog does not record their sizes); the per-account cap is set with that
// roughly-2x on-disk factor in mind
pub fn song_bytes(song: &JsonValue) -> i64 {
let mut total = song["jacket_size"].as_i64().unwrap_or(0) + song["jacket_blur_size"].as_i64().unwrap_or(0);
for level in song["levels"].members() {
total += level["size"].as_i64().unwrap_or(0);
}
for key in ["play", "select"] {
total += song["sound"][key]["size"].as_i64().unwrap_or(0);
}
total
}
// What this account's songs already occupy, optionally ignoring one song (the
// one being replaced by an in-place edit, whose new size is counted instead)
pub fn owner_bytes(owner_id: i64, excluding_music_id: i64) -> i64 {
let songs = DATABASE.lock_and_select_all("SELECT song FROM songs WHERE owner_id=?1", params!(owner_id)).unwrap_or(array![]);
let mut total = 0;
for blob in songs.members() {
let Ok(song) = jzon::parse(&blob.to_string()) else { continue; };
if song["music_id"].as_i64() == Some(excluding_music_id) {
continue;
}
total += song_bytes(&song);
}
total
}
// Whether the song exists and is publicly visible - what lets another
// uploader attach cross-feature content (a custom 3D MV) to it. The
// existence check comes first: get_visibility defaults to "public" for an
@@ -289,7 +365,10 @@ pub fn non_public_music_ids_for(user_id: i64) -> JsonValue {
// range is ever considered, so official songs can't come back from this. A song
// that's merely private/shared still has its row - only genuinely deleted ids
// (which are never reused) are returned
pub fn dead_music_ids(candidates: &JsonValue) -> JsonValue {
// None = the catalog could not be read (or is entirely empty while players still hold
// custom ids): the caller must prune NOTHING then. An unreadable catalog looks exactly
// like one where every id is dead, and get_acc saves the pruned userdata.
pub fn dead_music_ids(candidates: &JsonValue) -> Option<JsonValue> {
let mut ids: Vec<i64> = Vec::new();
for id in candidates.members() {
let Some(id) = id.as_i64() else { continue; };
@@ -298,17 +377,23 @@ pub fn dead_music_ids(candidates: &JsonValue) -> JsonValue {
}
}
if ids.is_empty() {
return array![];
return Some(array![]);
}
let list = ids.iter().map(|id| id.to_string()).collect::<Vec<_>>().join(",");
let alive = DATABASE.lock_and_select_all(&format!("SELECT music_id FROM songs WHERE music_id IN ({})", list), params!()).unwrap_or(array![]);
let alive = DATABASE.lock_and_select_all(&format!("SELECT music_id FROM songs WHERE music_id IN ({})", list), params!()).ok()?;
if alive.is_empty() {
let total: i64 = DATABASE.lock_and_select_type("SELECT COUNT(*) FROM songs", params!()).ok()?;
if total == 0 {
return None;
}
}
let mut rv = array![];
for id in ids {
if !alive.contains(id) {
rv.push(id).unwrap();
}
}
rv
Some(rv)
}
// Every stored catalog blob, unparsed and unfiltered by visibility. Only the
@@ -319,9 +404,32 @@ pub fn all_song_blobs() -> Option<JsonValue> {
DATABASE.lock_and_select_all("SELECT song FROM songs ORDER BY music_id", params!()).ok()
}
// Audio files are content-addressed and may be shared between songs
pub fn audio_in_use(md5: &str, ignored_music_id: i64) -> bool {
DATABASE.lock_and_select("SELECT music_id FROM songs WHERE music_id!=?1 AND song LIKE ?2", params!(ignored_music_id, format!("%{}%", md5))).is_ok()
// Audio files are content-addressed and may be shared between songs, so an ogg is
// only unlinkable once no other song's play/select cue names it. The LIKE scan finds
// candidate rows cheaply and the parsed cues confirm the hit (a substring coincidence
// elsewhere in the blob is not a reference), exactly like custom_3dmv::blob_in_use.
//
// Returns Result and NEVER folds an error into "false": the caller unlinks on false,
// and a read that failed (SQLITE_BUSY under a concurrent write, a corrupt page) says
// nothing about whether the file is referenced. The startup sweep is deliberately
// fail-closed for the same reason; this is the online half of that rule
pub fn audio_in_use(md5: &str, ignored_music_id: i64) -> Result<bool, rusqlite::Error> {
let rows = DATABASE.lock_and_select_all(
"SELECT song FROM songs WHERE music_id!=?1 AND song LIKE ?2",
params!(ignored_music_id, format!("%{}%", md5))
)?;
for blob in rows.members() {
let Ok(song) = jzon::parse(&blob.to_string()) else {
// An unparseable row could reference anything - assume it does
return Ok(true);
};
for key in ["play", "select"] {
if song["sound"][key]["md5"].as_str() == Some(md5) {
return Ok(true);
}
}
}
Ok(false)
}
// Resolve a content-addressed chart/jacket md5 to the per-song-id file that
@@ -348,3 +456,10 @@ pub fn find_asset_by_md5(md5: &str) -> Option<(i64, String)> {
}
None
}
// The on-disk database file, so a test can force the read errors the fail-closed
// GC paths are built for
#[cfg(test)]
pub fn test_db_path() -> String {
DATABASE.get_path().to_string()
}
+6
View File
@@ -86,6 +86,12 @@ pub struct Args {
#[arg(long, default_value = "", help = "Asset hash for windows client.")]
pub windows_asset_hash: String,
#[arg(long, default_value = "", help = "Asset hash for linux client.")]
pub linux_asset_hash: String,
#[arg(long, default_value = "", help = "Asset hash for macOS client.")]
pub macos_asset_hash: String,
#[arg(long, default_value = "", help = "Path to image assets.")]
pub image_asset_path: String,
+276 -18
View File
@@ -48,6 +48,19 @@ pub const MAX_FILE_BYTES: usize = 64 * 1024 * 1024;
pub const MAX_REQUEST_BYTES: usize = 256 * 1024 * 1024;
pub const MAX_MVS_PER_USER: i64 = 200;
// Per-account storage quota, counted over the stored file sizes the catalog
// quotes. The MV count alone bounded one account at 200 x 256MB = ~51GB, which is
// not a bound at all; 4GiB is roughly forty full MVs of realistic size (a model
// zip plus 20-100MB of motion VMDs) and is the largest of the three features'
// quotas because MVs are the heaviest thing a user can upload
pub const MAX_BYTES_PER_USER: i64 = 4 * 1024 * 1024 * 1024;
// The longest length-prefixed text field a PMX header may declare (model name and
// comment, JP + EN). The prefix is an attacker-supplied i32 the stage walk used to
// skip over by inflating up to 2GB into a sink, four times per entry - CPU with no
// allocation to show for it. Real PMX name/comment fields are tens of bytes
const MAX_PMX_TEXT_BYTES: i32 = 64 * 1024;
// Slots are 1-based, matching the Live3dMemberMst position convention
pub const MAX_MEMBER_COUNT: i64 = 12;
@@ -206,11 +219,11 @@ async fn read_multipart(mut payload: Multipart) -> Result<Fields, String> {
while let Some(chunk) = field.try_next().await.map_err(|e| e.to_string())? {
total += chunk.len();
if total > MAX_REQUEST_BYTES {
return Err(format!("Upload exceeds the {} MB per-request limit", MAX_REQUEST_BYTES / (1024 * 1024)));
return Err(over_request_limit());
}
data.extend_from_slice(&chunk);
if data.len() > MAX_FILE_BYTES {
return Err(format!("'{}' exceeds the {} MB per-file limit", name, MAX_FILE_BYTES / (1024 * 1024)));
return Err(over_file_limit(&name));
}
}
fields.insert(name, data);
@@ -218,6 +231,32 @@ async fn read_multipart(mut payload: Multipart) -> Result<Fields, String> {
Ok(fields)
}
pub fn over_file_limit(name: &str) -> String {
format!("'{}' exceeds the {} MB per-file limit", name, MAX_FILE_BYTES / (1024 * 1024))
}
pub fn over_request_limit() -> String {
format!("Upload exceeds the {} MB per-request limit", MAX_REQUEST_BYTES / (1024 * 1024))
}
// The same accounting read_multipart applies, re-run over a field map that came
// out of an export package. package::expand caps every entry as it inflates, but
// the caps have to hold over the RESULT too: a package is one multipart field and
// its expansion replaces the whole form
fn check_field_caps(fields: &Fields) -> Result<(), String> {
let mut total = 0usize;
for (name, data) in fields.iter() {
if data.len() > MAX_FILE_BYTES {
return Err(over_file_limit(name));
}
total += data.len();
if total > MAX_REQUEST_BYTES {
return Err(over_request_limit());
}
}
Ok(())
}
fn field_str(fields: &Fields, key: &str) -> String {
String::from_utf8_lossy(fields.get(key).map(|v| v.as_slice()).unwrap_or(&[])).trim().to_string()
}
@@ -263,7 +302,7 @@ fn read_pmx_vertex_count(file: &mut impl Read) -> Option<i32> {
let mut len = [0u8; 4];
file.read_exact(&mut len).ok()?;
let len = i32::from_le_bytes(len);
if len < 0 {
if !(0..=MAX_PMX_TEXT_BYTES).contains(&len) {
return None;
}
if std::io::copy(&mut file.by_ref().take(len as u64), &mut std::io::sink()).ok()? != len as u64 {
@@ -436,7 +475,9 @@ fn write_blobs(pending: &[PendingBlob]) -> Result<(), String> {
fn gc_blobs(old_files: &JsonValue) {
for file in old_files.members() {
let md5 = file["md5"].to_string();
if md5.len() == 32 && !database::blob_in_use(&md5) {
// A read error means "assume referenced": unlinking on a doubtful
// reference set is exactly what the startup sweep refuses to do
if md5.len() == 32 && !database::blob_in_use(&md5).unwrap_or(true) {
let _ = fs::remove_file(blob_path(&md5));
}
}
@@ -477,6 +518,7 @@ pub fn create_mv(uid: i64, fields: &Fields) -> Result<i64, String> {
}
let (files, pending) = collect_files(fields, member_count, &array![])?;
check_quota(uid, database::mv_bytes(&files), 0)?;
let lock = lock_onto_mutex!(UPLOAD_LOCK);
let mv_id = database::next_mv_id();
@@ -494,13 +536,27 @@ pub fn create_mv(uid: i64, fields: &Fields) -> Result<i64, String> {
};
write_blobs(&pending)?;
database::insert_mv(mv_id, music_id, uid, &mv, published);
database::insert_mv(mv_id, music_id, uid, &mv, published)
.map_err(|e| format!("Could not store the MV: {}", e))?;
database::bump_revision();
drop(lock);
Ok(mv_id)
}
// Per-account storage quota. `excluded` is the MV being replaced by an in-place
// edit - its stored size drops out and `adding` (the resulting size) replaces it
fn check_quota(uid: i64, adding: i64, excluded_mv_id: i64) -> Result<(), String> {
let used = database::owner_bytes(uid, excluded_mv_id);
if used + adding > MAX_BYTES_PER_USER {
return Err(format!(
"This upload would put your MVs at {} MB, over the {} MB per-account limit - delete an MV first",
(used + adding) / (1024 * 1024), MAX_BYTES_PER_USER / (1024 * 1024)
));
}
Ok(())
}
// Edit an MV in place. The mv_id and the music_id stay the same: repointing
// the song would break the catalog closure for everyone who already resolved
// it (delete + re-upload retires the id instead)
@@ -523,6 +579,7 @@ pub fn update_mv(uid: i64, mv_id: i64, fields: &Fields) -> Result<(), String> {
}
let (files, pending) = collect_files(fields, member_count, &stored["files"])?;
check_quota(owner, database::mv_bytes(&files), mv_id)?;
let mv = object!{
"mv_id": mv_id,
@@ -598,6 +655,33 @@ pub fn purge_song(music_id: i64) {
drop(lock);
}
// Every MV this account uploaded, gone - called from userdata::delete_account, so
// a purged uploader leaves no catalog row resolving an owner id that no longer
// exists (browse renders an uploader name for every row). Same steps as the
// owner's own delete, blob GC included
pub fn purge_owner(uid: i64) {
if uid <= 0 {
return;
}
if disabled() {
return;
}
let lock = lock_onto_mutex!(UPLOAD_LOCK);
let mut purged = false;
for mv_id in database::mv_ids_for_owner(uid) {
let stored = database::get_mv(mv_id);
database::delete_mv(mv_id);
purged = true;
if let Some(stored) = stored {
gc_blobs(&stored["files"]);
}
}
if purged {
database::bump_revision();
}
drop(lock);
}
// Startup GC for the content-addressed blob store, mirroring
// custom_song::sweep_audio: the only writers are upload and update, so an
// unreferenced file is a leftover from an interrupted one. Deliberately
@@ -634,6 +718,12 @@ pub fn sweep_blobs() {
}
}
// Nothing referenced means the catalog is empty (or gone): a sweep would then delete
// every blob on disk, so it is skipped until the catalog has rows again
if referenced.is_empty() {
println!("Custom 3DMV blob sweep: catalog empty, skipped");
return;
}
// No directory means nothing was ever uploaded
let Ok(entries) = fs::read_dir(get_data_path("custom_3dmv/blobs")) else {
return;
@@ -666,6 +756,7 @@ pub fn upload_limits() -> JsonValue {
"max_file_bytes": MAX_FILE_BYTES,
"max_request_bytes": MAX_REQUEST_BYTES,
"max_mvs_per_user": MAX_MVS_PER_USER,
"max_bytes_per_user": MAX_BYTES_PER_USER,
"stages": STAGES.to_vec(),
"default_stage": STAGES[0],
"roles": {
@@ -690,21 +781,29 @@ async fn upload(req: HttpRequest, payload: Multipart) -> HttpResponse {
Ok(fields) => fields,
Err(e) => return webui::error(&e)
};
// Zip inflation, the PMX/VMD structure walks, hashing and writing up to 256MB:
// all of it on the blocking pool rather than on the actix worker that also has
// to keep serving the game API
let result = web::block(move || {
// An export package from another server: its contents map 1:1 onto the
// normal upload fields, so importing is just an upload
if let Some(bytes) = fields.remove("package") {
if !bytes.is_empty() {
if let Err(e) = package::expand(&bytes, &mut fields) {
return webui::error(&e);
package::expand(&bytes, &mut fields)?;
// The expansion replaced the form: it has to satisfy the same
// per-file/per-request caps the multipart reader enforces
check_field_caps(&fields)?;
}
}
}
match create_mv(uid, &fields) {
Ok(mv_id) => send_json(object!{
create_mv(uid, &fields)
}).await;
match result {
Ok(Ok(mv_id)) => send_json(object!{
result: "OK",
mv_id: mv_id
}),
Err(e) => webui::error(&e)
Ok(Err(e)) => webui::error(&e),
Err(_) => webui::error("The upload could not be processed")
}
}
@@ -720,12 +819,13 @@ async fn update(req: HttpRequest, payload: Multipart) -> HttpResponse {
Err(e) => return webui::error(&e)
};
let mv_id = field_str(&fields, "mv_id").parse::<i64>().unwrap_or(0);
match update_mv(uid, mv_id, &fields) {
Ok(()) => send_json(object!{
match web::block(move || update_mv(uid, mv_id, &fields)).await {
Ok(Ok(())) => send_json(object!{
result: "OK",
mv_id: mv_id
}),
Err(e) => webui::error(&e)
Ok(Err(e)) => webui::error(&e),
Err(_) => webui::error("The edit could not be processed")
}
}
@@ -806,7 +906,15 @@ async fn download(req: HttpRequest) -> HttpResponse {
let Some(owner) = database::get_mv_owner(mv_id) else {
return webui::error("MV not found");
};
if get_session_uid(&req) != Some(owner) && !database::is_published(mv_id) {
let viewer = get_session_uid(&req);
if viewer != Some(owner) && !database::is_published(mv_id) {
return webui::error("MV not found");
}
// Every catalog (list, browse, mine) additionally closes over the songs the
// viewer can see, so a published MV attached to someone else's PRIVATE song is
// invisible everywhere - it must not be downloadable by walking mv_ids either
let music_id = database::get_mv_music_id(mv_id).unwrap_or(0);
if viewer != Some(owner) && !allowed_music_ids(viewer.unwrap_or(0)).contains(&music_id) {
return webui::error("MV not found");
}
match package::build(mv_id) {
@@ -940,7 +1048,7 @@ pub mod tests {
"music_id": music_id,
"name": format!("Seed Song {}", music_id),
"sound": { "play": { "md5": "0".repeat(32) }, "select": { "md5": "0".repeat(32) } }
}, visibility, &array![], false);
}, visibility, &array![], false).unwrap();
}
// A complete, valid 2-slot upload: model+motion per slot, a facial on
@@ -1420,9 +1528,9 @@ pub mod tests {
assert!(!stranger_catalog.members().any(|m| m["mv_id"] == private_song_mv));
// Sharing the song brings its MV along
crate::database::custom_song::set_visibility(970012, "shared", &array![stranger]);
crate::database::custom_song::set_visibility(970012, "shared", &array![stranger]).unwrap();
assert!(catalog_for_user(stranger).members().any(|m| m["mv_id"] == private_song_mv));
crate::database::custom_song::set_visibility(970012, "private", &array![]);
crate::database::custom_song::set_visibility(970012, "private", &array![]).unwrap();
wipe(owner);
}
@@ -1501,4 +1609,154 @@ pub mod tests {
let _ = fs::remove_file(&junk);
wipe(uid);
}
// ---- defect-fix coverage -------------------------------------------------
// A stage PMX whose header declares a text field of `len` bytes, with the
// bytes actually present
fn stage_zip_with_text_len(len: i32, present: usize) -> Vec<u8> {
let mut pmx = Vec::new();
pmx.extend(b"PMX ");
pmx.extend(2.0f32.to_le_bytes());
pmx.push(8);
pmx.extend([0u8; 8]);
// The model name carries the declared length; the other three are empty
pmx.extend(len.to_le_bytes());
pmx.extend(vec![0u8; present]);
for _ in 0..3 {
pmx.extend(0u32.to_le_bytes());
}
pmx.extend(1i32.to_le_bytes());
zip_with("stage.pmx", &pmx)
}
// D11: the length prefix of a PMX text field is an attacker-supplied i32 the
// stage walk skips over. Uncapped it inflated up to 2GB into a sink, four
// times per entry - CPU with nothing to show for it
#[test]
fn pmx_text_fields_are_bounded() {
let big = MAX_PMX_TEXT_BYTES as usize;
// At the cap, with the bytes really there: a valid stage
assert!(validate_file("stage", "stage", &stage_zip_with_text_len(big as i32, big)).is_ok());
// One byte over, bytes present: refused rather than skipped over
let err = validate_file("stage", "stage", &stage_zip_with_text_len(big as i32 + 1, big + 1)).unwrap_err();
assert!(err.contains("malformed PMX header"), "{}", err);
// And the classic: a huge declaration with nothing behind it
let err = validate_file("stage", "stage", &stage_zip_with_text_len(i32::MAX, 0)).unwrap_err();
assert!(err.contains("malformed PMX header"), "{}", err);
}
// D1: a package entry is capped as it inflates. Deflate's ~1032:1 ceiling
// means an uncapped read_to_end turns a tiny zip into gigabytes, and a package
// carries 39 addressable entries
#[test]
fn package_import_is_capped() {
let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));
let options = zip::write::SimpleFileOptions::default();
zip.start_file("manifest.json", options).unwrap();
zip.write_all(jzon::stringify(object!{
"format": 1, "name": "Bomb", "name_en": "Bomb", "music_id": 970050, "member_count": 1
}).as_bytes()).unwrap();
// Compresses to a few KB, inflates to just over the per-file cap
zip.start_file("model_1", options).unwrap();
zip.write_all(&vec![0u8; MAX_FILE_BYTES + 1]).unwrap();
let package = zip.finish().unwrap().into_inner();
assert!(package.len() < 1024 * 1024, "the bomb should be small: {} bytes", package.len());
let mut fields = Fields::new();
let err = package::expand(&package, &mut fields).unwrap_err();
assert!(err.contains("per-file limit"), "{}", err);
assert!(fields.get("model_1").is_none(), "the oversized entry was buffered anyway");
}
// D1/D6: the per-request total is accounted over the whole form, which is what
// a package's expanded contents are re-checked against
#[test]
fn the_request_total_is_capped() {
let mut fields = Fields::new();
fields.insert(String::from("a"), vec![0; MAX_FILE_BYTES]);
assert!(check_field_caps(&fields).is_ok());
let mut one = Fields::new();
one.insert(String::from("model_1"), vec![0; MAX_FILE_BYTES + 1]);
assert!(check_field_caps(&one).unwrap_err().contains("per-file limit"));
}
// D2: a read error must never read as "not referenced". The GC unlinks on
// false, so a failed lookup has to mean "assume in use" - the startup sweep
// has always been fail-closed and the online path now matches it
#[test]
fn a_database_error_never_deletes_a_blob() {
let _lock = crate::runtime::lock_test_data_path();
wipe(9_100_030);
seed_song(970201, 9_100_030, "public");
let id = create_mv(9_100_030, &base_fields(970201, 1, 120)).unwrap();
let stored = database::get_mv(id).unwrap();
let md5 = stored["files"][0]["md5"].to_string();
assert_eq!(md5.len(), 32);
assert_eq!(database::blob_in_use(&md5), Ok(true));
assert_eq!(database::blob_in_use(&"c7".repeat(16)), Ok(false));
let conn = rusqlite::Connection::open(database::test_db_path()).unwrap();
conn.execute("ALTER TABLE mvs RENAME TO mvs_hidden", ()).unwrap();
assert!(database::blob_in_use(&md5).is_err());
// The blob is still live; the GC must keep what it cannot prove is orphaned
gc_blobs(&stored["files"]);
conn.execute("ALTER TABLE mvs_hidden RENAME TO mvs", ()).unwrap();
assert!(fs::read(blob_path(&md5)).is_ok(), "a live blob was unlinked on a read error");
wipe(9_100_030);
}
// D9: every catalog closes over the songs the viewer can see, so a published
// MV attached to someone else's PRIVATE song is invisible everywhere - and it
// must not be downloadable by walking mv_ids either
#[test]
fn download_closes_over_song_visibility() {
let _lock = crate::runtime::lock_test_data_path();
wipe(9_100_031);
seed_song(970202, 9_100_031, "private");
seed_song(970203, 9_100_031, "public");
let hidden = create_mv(9_100_031, &base_fields(970202, 1, 130)).unwrap();
let open = create_mv(9_100_031, &base_fields(970203, 1, 140)).unwrap();
set_mv_flags(9_100_031, hidden, true).unwrap();
set_mv_flags(9_100_031, open, true).unwrap();
let call = |mv_id: i64| -> String {
let req = actix_web::test::TestRequest::default().param("mv_id", mv_id.to_string()).to_http_request();
actix_web::rt::System::new().block_on(async {
let resp = download(req).await;
let bytes = actix_web::body::to_bytes(resp.into_body()).await.unwrap();
String::from_utf8_lossy(&bytes).to_string()
})
};
// Published, on a song an anonymous viewer's catalog never delivers
assert!(call(hidden).contains("MV not found"), "a private song's MV was downloadable");
// Published, on a public song: still downloadable
assert!(!call(open).contains("MV not found"));
wipe(9_100_031);
}
// D15: the quota counts the stored file bytes the catalog quotes, per owner
#[test]
fn uploads_are_bounded_by_a_per_account_byte_quota() {
let _lock = crate::runtime::lock_test_data_path();
wipe(9_100_032);
assert_eq!(database::owner_bytes(9_100_032, 0), 0);
seed_song(970204, 9_100_032, "public");
let id = create_mv(9_100_032, &base_fields(970204, 1, 150)).unwrap();
let stored = database::mv_bytes(&database::get_mv(id).unwrap()["files"]);
assert!(stored > 0);
assert_eq!(database::owner_bytes(9_100_032, 0), stored);
// An in-place edit replaces its own bytes rather than adding to them
assert_eq!(database::owner_bytes(9_100_032, id), 0);
assert!(check_quota(9_100_032, 1, 0).is_ok());
assert!(check_quota(9_100_032, MAX_BYTES_PER_USER, 0).unwrap_err().contains("per-account limit"));
assert_eq!(database::owner_bytes(9_100_033, 0), 0);
wipe(9_100_032);
}
}
+38 -7
View File
@@ -42,11 +42,40 @@ pub fn build(mv_id: i64) -> Result<Vec<u8>, String> {
Ok(zip.finish().map_err(|e| e.to_string())?.into_inner())
}
fn read_entry<R: Read + Seek>(archive: &mut zip::ZipArchive<R>, name: &str) -> Option<Vec<u8>> {
let mut file = archive.by_name(name).ok()?;
// Reads one entry, capped. Deflate's ceiling is about 1032:1, so an uncapped
// read_to_end here is a zip bomb: a one-megabyte entry inflates to a gigabyte and
// grows the Vec until the allocator or the OOM killer stops it, and a package has
// 39 addressable entries. Every entry is bounded by the upload form's own
// per-file cap, and by what is left of the per-request budget across all entries.
//
// The central directory's declared size rejects the obvious case without
// inflating anything; take(cap + 1) makes that declaration untrusted - a lying
// header runs out of budget one byte past the cap and stops there.
//
// Ok(None) is "the package does not carry this entry", a normal outcome for every
// optional role
fn read_entry<R: Read + Seek>(archive: &mut zip::ZipArchive<R>, name: &str, remaining: &mut usize) -> Result<Option<Vec<u8>>, String> {
let Ok(mut file) = archive.by_name(name) else {
return Ok(None);
};
let cap = std::cmp::min(super::MAX_FILE_BYTES, *remaining);
// Which limit the entry actually ran into, so the message names the right one
let too_big = if cap >= super::MAX_FILE_BYTES {
super::over_file_limit(name)
} else {
super::over_request_limit()
};
if file.size() > cap as u64 {
return Err(too_big);
}
let mut bytes = Vec::new();
file.read_to_end(&mut bytes).ok()?;
Some(bytes)
file.by_ref().take(cap as u64 + 1).read_to_end(&mut bytes)
.map_err(|_| format!("Package entry '{}' could not be read", name))?;
if bytes.len() > cap {
return Err(too_big);
}
*remaining -= bytes.len();
Ok(Some(bytes))
}
// Expands a package into the same field map the upload form produces. The
@@ -55,8 +84,10 @@ fn read_entry<R: Read + Seek>(archive: &mut zip::ZipArchive<R>, name: &str) -> O
// in when the form left it blank (same-server re-upload)
pub fn expand(package: &[u8], fields: &mut HashMap<String, Vec<u8>>) -> Result<(), String> {
let mut archive = zip::ZipArchive::new(Cursor::new(package)).map_err(|_| String::from("Package is not a valid zip file"))?;
// The decompressed budget for the whole package, shared by every entry
let mut remaining = super::MAX_REQUEST_BYTES;
let manifest = read_entry(&mut archive, "manifest.json").ok_or(String::from("Package has no manifest.json"))?;
let manifest = read_entry(&mut archive, "manifest.json", &mut remaining)?.ok_or(String::from("Package has no manifest.json"))?;
let manifest = jzon::parse(&String::from_utf8_lossy(&manifest)).map_err(|_| String::from("Package manifest is not valid JSON"))?;
if manifest["format"].as_i64() != Some(1) {
return Err(String::from("Unsupported package format"));
@@ -74,13 +105,13 @@ pub fn expand(package: &[u8], fields: &mut HashMap<String, Vec<u8>>) -> Result<(
let member_count = manifest["member_count"].as_i64().unwrap_or(0);
for slot in 1..=member_count.clamp(0, super::MAX_MEMBER_COUNT) {
for role in ["model", "motion", "facial"] {
if let Some(bytes) = read_entry(&mut archive, &format!("{}_{}", role, slot)) {
if let Some(bytes) = read_entry(&mut archive, &format!("{}_{}", role, slot), &mut remaining)? {
fields.insert(format!("{}_{}", role, slot), bytes);
}
}
}
for name in ["camera", "config", "stage"] {
if let Some(bytes) = read_entry(&mut archive, name) {
if let Some(bytes) = read_entry(&mut archive, name, &mut remaining)? {
fields.insert(String::from(name), bytes);
}
}
+478 -25
View File
@@ -52,6 +52,13 @@ pub const MAX_FILE_BYTES: usize = 8 * 1024 * 1024;
pub const MAX_REQUEST_BYTES: usize = 64 * 1024 * 1024;
pub const MAX_CARDS_PER_USER: i64 = 500;
// Per-account storage quota over the stored art and voiceline bytes, shared by
// cards and characters (they share one storage root). One card derives 14 PNGs up
// to 2048x1260, so the 500-card count limit alone allowed several gigabytes per
// account with no byte bound at all; 2GiB is around a hundred full cards, well
// past any real uploader, and matches the custom-song quota
pub const MAX_BYTES_PER_USER: i64 = 2 * 1024 * 1024 * 1024;
// Columns the uploader never supplies. master_release_label_id must be 1: a
// closed label filters the card out of the member-list filters and drops its
// evolve conditions
@@ -127,6 +134,24 @@ const CHARACTER_ART: &[ArtKind] = &[
ArtKind { kind: "character", width: 600, height: 920 }
];
// One uploadable voiceline moment. `name` is the wire kind: it is the infix of
// the multipart field names below, the "kind" of every stored/served line, and
// what the client maps to a SYSTEM_VOICE_TYPE (and, for the day_* moments, to
// the date_condition id the game's resolver compares). `category` only groups
// the upload form.
//
// The home-screen moments (category "home") are exactly the SYSTEM_VOICE_TYPEs
// HomeScene.CreateVoiceData asks the resolver for; everything else is a moment
// that already shipped. Kind names are the snake_case of the game's own
// SYSTEM_VOICE_TYPE members (day_* follow the game's own SYS_VOICE_DAY_MMDD cue
// naming), so the mapping stays readable on both sides.
pub struct VoiceKind {
pub name: &'static str,
pub label: &'static str,
pub label_en: &'static str,
pub category: &'static str,
}
// Optional voicelines per character. Multipart fields per line:
// voice_{kind}_{index} audio file (any format symphonia reads;
// transcoded to ogg-vorbis, stored
@@ -137,9 +162,71 @@ const CHARACTER_ART: &[ArtKind] = &[
// Absent slots keep their stored line, captions without a file update the
// stored line's captions, and surviving lines are renumbered contiguously
// per kind (1..n) after every edit
const VOICE_KINDS: &[&str] = &[
"live_start", "live_success", "live_failed", "result_bond",
"skill_smile", "skill_pure", "skill_cool"
pub const VOICE_KINDS: &[VoiceKind] = &[
// --- moments outside the home screen (unchanged wire names) ---
VoiceKind { name: "live_start", label: "ライブ開始(リーダー)", label_en: "Live start (leader, before a live)", category: "live" },
VoiceKind { name: "live_success", label: "ライブクリア", label_en: "Live cleared", category: "live" },
VoiceKind { name: "live_failed", label: "ライブ失敗", label_en: "Live failed", category: "live" },
VoiceKind { name: "result_bond", label: "リザルト・絆獲得", label_en: "Bond gained (live result)", category: "result" },
VoiceKind { name: "skill_smile", label: "スキル発動(スマイル)", label_en: "Skill activation - Smile cards", category: "skill" },
VoiceKind { name: "skill_pure", label: "スキル発動(ピュア)", label_en: "Skill activation - Pure cards", category: "skill" },
VoiceKind { name: "skill_cool", label: "スキル発動(クール)", label_en: "Skill activation - Cool cards", category: "skill" },
// --- home screen: always in the pool ---
VoiceKind { name: "random", label: "ランダム", label_en: "Random line (always in the pool)", category: "home" },
VoiceKind { name: "random_evolution", label: "ランダム(覚醒後)", label_en: "Random line, awakened card only", category: "home" },
VoiceKind { name: "random_unevolution_smile", label: "ランダム(未覚醒・スマイル)", label_en: "Random line, un-awakened Smile card", category: "home" },
VoiceKind { name: "random_unevolution_pure", label: "ランダム(未覚醒・ピュア)", label_en: "Random line, un-awakened Pure card", category: "home" },
VoiceKind { name: "random_unevolution_cool", label: "ランダム(未覚醒・クール)", label_en: "Random line, un-awakened Cool card", category: "home" },
VoiceKind { name: "random_evolution_smile", label: "ランダム(覚醒後・スマイル)", label_en: "Random line, awakened Smile card", category: "home" },
VoiceKind { name: "random_evolution_pure", label: "ランダム(覚醒後・ピュア)", label_en: "Random line, awakened Pure card", category: "home" },
VoiceKind { name: "random_evolution_cool", label: "ランダム(覚醒後・クール)", label_en: "Random line, awakened Cool card", category: "home" },
VoiceKind { name: "touch", label: "タップ", label_en: "Tapped (the touch-to-play-voice button)", category: "home" },
// --- home screen: time of day (the game's own hour bands) ---
VoiceKind { name: "time", label: "時間帯(共通)", label_en: "Any time of day", category: "home" },
VoiceKind { name: "time_firsthalf", label: "時間帯(5時〜17時)", label_en: "First half of the day (05-17)", category: "home" },
VoiceKind { name: "time_latterhalf", label: "時間帯(17時〜5時)", label_en: "Latter half of the day (17-05)", category: "home" },
VoiceKind { name: "time_morning", label: "時間帯(朝・5時〜11時)", label_en: "Morning (05-11)", category: "home" },
VoiceKind { name: "time_noon", label: "時間帯(昼・11時〜17時)", label_en: "Daytime (11-17)", category: "home" },
VoiceKind { name: "time_evening", label: "時間帯(夕方・17時〜23時)", label_en: "Evening (17-23)", category: "home" },
VoiceKind { name: "time_night", label: "時間帯(夜・23時〜5時)", label_en: "Night (23-05)", category: "home" },
// --- home screen: season (the game's own month bands) ---
VoiceKind { name: "season_spring", label: "季節(春・3〜5月)", label_en: "Spring (March-May)", category: "home" },
VoiceKind { name: "season_summer", label: "季節(夏・6〜8月)", label_en: "Summer (June-August)", category: "home" },
VoiceKind { name: "season_autumn", label: "季節(秋・9〜11月)", label_en: "Autumn (September-November)", category: "home" },
VoiceKind { name: "season_winter", label: "季節(冬・12〜2月)", label_en: "Winter (December-February)", category: "home" },
// --- home screen: calendar days. On a day that matches, the home screen
// asks for NOTHING ELSE (a matching day line pre-empts the whole pool
// unless the player taps), so these are the "special occasion" lines
VoiceKind { name: "day_0101", label: "記念日(1月1日・お正月)", label_en: "January 1 (New Year's Day)", category: "home" },
VoiceKind { name: "day_0203", label: "記念日(2月3日・節分)", label_en: "February 3 (Setsubun)", category: "home" },
VoiceKind { name: "day_0214", label: "記念日(2月14日・バレンタイン)", label_en: "February 14 (Valentine's Day)", category: "home" },
VoiceKind { name: "day_0303", label: "記念日(3月3日・ひな祭り)", label_en: "March 3 (Hinamatsuri)", category: "home" },
VoiceKind { name: "day_0314", label: "記念日(3月14日・ホワイトデー)", label_en: "March 14 (White Day)", category: "home" },
VoiceKind { name: "day_0505", label: "記念日(5月5日・こどもの日)", label_en: "May 5 (Children's Day)", category: "home" },
VoiceKind { name: "day_0707", label: "記念日(7月7日・七夕)", label_en: "July 7 (Tanabata)", category: "home" },
VoiceKind { name: "day_0717", label: "記念日(7月第3月曜・海の日)", label_en: "Marine Day (third Monday of July)", category: "home" },
VoiceKind { name: "day_1015", label: "記念日(10月15日)", label_en: "October 15", category: "home" },
VoiceKind { name: "day_1031", label: "記念日(10月31日・ハロウィン)", label_en: "October 31 (Halloween)", category: "home" },
VoiceKind { name: "day_1225", label: "記念日(12月25日・クリスマス)", label_en: "December 25 (Christmas)", category: "home" },
VoiceKind { name: "day_1231", label: "記念日(12月31日・大晦日)", label_en: "December 31 (New Year's Eve)", category: "home" },
// --- home screen: lines keyed to what the account has waiting ---
VoiceKind { name: "advice_story", label: "未読ストーリーあり(おすすめ)", label_en: "Unread story waiting (advice line)", category: "home" },
VoiceKind { name: "trigger_story", label: "未読ストーリーあり(反応)", label_en: "Unread story waiting (trigger line)", category: "home" },
VoiceKind { name: "advice_mission", label: "デイリーミッション報酬未受取", label_en: "Unclaimed daily mission reward", category: "home" },
VoiceKind { name: "advice_live", label: "LPが足りている(ライブ)", label_en: "Enough LP to play a live", category: "home" },
VoiceKind { name: "advice_lp", label: "LP半分以下(回復のすすめ)", label_en: "LP below half, recovery affordable", category: "home" },
VoiceKind { name: "trigger_shop", label: "LP半分以下(ショップ)", label_en: "LP below half, recovery affordable (shop line)", category: "home" },
VoiceKind { name: "advice_lesson", label: "レッスン(未カンスト)", label_en: "A unit is not fully levelled", category: "home" },
VoiceKind { name: "advice_present", label: "プレゼント未受取", label_en: "Unclaimed present waiting", category: "home" },
VoiceKind { name: "advice_gacha", label: "無料スカウト可能", label_en: "The daily free scouting is available", category: "home" },
VoiceKind { name: "advice_infomation", label: "新着お知らせあり", label_en: "A new announcement is up", category: "home" },
VoiceKind { name: "advice_event", label: "イベント開催中", label_en: "An event is running", category: "home" },
VoiceKind { name: "trigger_friend", label: "イベント中・フレンド", label_en: "An event is running and a friend is active", category: "home" }
];
const MAX_VOICE_VARIANTS: usize = 9;
const MAX_VOICE_BYTES: usize = 4 * 1024 * 1024;
@@ -482,6 +569,17 @@ pub fn upload_limits() -> JsonValue {
};
}
let ((prob_min, prob_max), (ms_min, ms_max)) = *SKILL_SCALAR_RANGES;
// The voiceline moments, so the form renders exactly the kinds this build
// accepts instead of carrying its own copy of the list
let mut voice_kinds = array![];
for kind in VOICE_KINDS {
voice_kinds.push(object!{
"name": kind.name,
"label": kind.label,
"label_en": kind.label_en,
"category": kind.category
}).unwrap();
}
object!{
"stat_caps": stat_caps,
"skill_levels": skill_levels,
@@ -498,7 +596,15 @@ pub fn upload_limits() -> JsonValue {
"probability": { "min": prob_min, "max": prob_max },
"milli_secs": { "min": ms_min, "max": ms_max }
},
"min_source_dim": art::MIN_SOURCE_DIM
"min_source_dim": art::MIN_SOURCE_DIM,
"max_file_bytes": MAX_FILE_BYTES,
"max_request_bytes": MAX_REQUEST_BYTES,
"max_cards_per_user": MAX_CARDS_PER_USER,
"max_bytes_per_user": MAX_BYTES_PER_USER,
"voice_kinds": voice_kinds,
"max_voice_variants": MAX_VOICE_VARIANTS,
"max_voice_bytes": MAX_VOICE_BYTES,
"max_voice_seconds": MAX_VOICE_SECONDS
}
}
@@ -559,6 +665,17 @@ async fn data(req: HttpRequest) -> HttpResponse {
return HttpResponse::NotFound().finish();
};
match fs::read(asset_path(&relative)) {
// Art lives at FIXED per-card filenames ({kind}_{variant}.png) that an
// in-place edit overwrites, so between the file write and the catalog
// update the index still points an old md5 at bytes that are no longer its
// own. The client caches whatever it downloads under the md5 it asked for
// and never re-checks, so serving those bytes would poison its cache
// permanently. The index is a hint; these bytes are the answer only if
// they hash to the request. A mismatch is the same 404 a stale md5 already
// gets, and the client re-downloads under the md5 the catalog now carries
Ok(body) if !hash.eq_ignore_ascii_case(&format!("{:x}", md5::compute(&body))) => {
HttpResponse::NotFound().finish()
},
Ok(body) => {
HttpResponse::Ok()
.insert_header(ContentType::png())
@@ -840,11 +957,29 @@ fn write_art(dir: &str, pending: &[PendingArt]) -> Result<(), String> {
fn collect_voice(fields: &Fields, stored_voice: &JsonValue) -> Result<(JsonValue, Vec<(String, Vec<u8>)>), String> {
let mut rv = array![];
let mut files: Vec<(String, Vec<u8>)> = Vec::new();
// An unknown moment is a typo, and silently ignoring it (what happens to
// every other unrecognised multipart field) would show the uploader a clip
// that uploaded fine and a line that never appeared. An out-of-range INDEX
// stays ignored - that is a full slot list, not a misspelling
for key in fields.keys() {
let Some(rest) = key.strip_prefix("voice_") else { continue; };
let rest = ["_text_en", "_text", "_delete"].iter()
.find_map(|suffix| rest.strip_suffix(suffix))
.unwrap_or(rest);
let named_moment = match rest.rsplit_once('_') {
Some((kind, index)) => index.parse::<usize>().is_ok() && VOICE_KINDS.iter().any(|k| k.name == kind),
None => false
};
if !named_moment {
return Err(format!("'{}' is not a voiceline field", key));
}
}
for kind in VOICE_KINDS {
let kind = kind.name;
let mut lines: Vec<JsonValue> = Vec::new();
for index in 1..=MAX_VOICE_VARIANTS {
let base = format!("voice_{}_{}", kind, index);
let stored_line = stored_voice.members().find(|line| line["kind"] == *kind && line["index"] == index);
let stored_line = stored_voice.members().find(|line| line["kind"] == kind && line["index"] == index);
if field_flag(fields, &format!("{}_delete", base)) {
if file_of(fields, &base).is_some() {
return Err(format!("'{}': cannot both replace and delete the same line", base));
@@ -869,7 +1004,7 @@ fn collect_voice(fields: &Fields, stored_voice: &JsonValue) -> Result<(JsonValue
}
let clip = audio::process_one_shot(bytes, MAX_VOICE_SECONDS).map_err(|e| format!("'{}': {}", base, e))?;
lines.push(object!{
"kind": *kind,
"kind": kind,
"index": 0,
"md5": clip.md5.clone(),
"size": clip.bytes.len(),
@@ -879,7 +1014,7 @@ fn collect_voice(fields: &Fields, stored_voice: &JsonValue) -> Result<(JsonValue
files.push((clip.md5, clip.bytes));
} else if let Some(stored_line) = stored_line {
lines.push(object!{
"kind": *kind,
"kind": kind,
"index": 0,
"md5": stored_line["md5"].clone(),
"size": stored_line["size"].clone(),
@@ -917,9 +1052,12 @@ fn write_voice(master_character_id: i64, files: &[(String, Vec<u8>)]) -> Result<
// (their old catalog md5 404s, exactly like replaced art)
fn gc_voice(master_character_id: i64, old_voice: &JsonValue, new_voice: &JsonValue) {
for old in old_voice.members() {
let md5 = old["md5"].to_string();
if !md5.is_empty() && !new_voice.members().any(|line| line["md5"] == old["md5"]) {
let _ = fs::remove_file(voice_path(master_character_id, &md5));
// JsonValue::to_string renders Null as the literal "null", which an
// is_empty() guard happily passes; only exactly 32 hex characters is a
// hash (the same test custom_3dmv's blob GC uses)
let Some(md5) = old["md5"].as_str().filter(|md5| md5.len() == 32 && md5.chars().all(|c| c.is_ascii_hexdigit())) else { continue; };
if !new_voice.members().any(|line| line["md5"] == old["md5"]) {
let _ = fs::remove_file(voice_path(master_character_id, md5));
}
}
}
@@ -1215,15 +1353,31 @@ pub fn create_card(uid: i64, fields: &Fields) -> Result<i64, String> {
let mut card = build_card(master_card_id, master_character_id, fields, &object!{})?;
card["art"] = merge_art(&array![], &card_art);
check_quota(uid, database::card_bytes(&card), 0, 0)?;
write_art(&card_dir(master_card_id), &card_art)?;
database::insert_card(master_card_id, master_character_id, uid, &card, published, obtainable);
database::insert_card(master_card_id, master_character_id, uid, &card, published, obtainable)
.map_err(|e| format!("Could not store the card: {}", e))?;
database::bump_revision();
drop(lock);
Ok(master_card_id)
}
// Per-account storage quota over this account's cards AND characters. The
// excluded ids are the entity being replaced by an in-place edit - its stored
// size drops out and `adding` (the resulting size) replaces it
fn check_quota(uid: i64, adding: i64, excluded_card_id: i64, excluded_character_id: i64) -> Result<(), String> {
let used = database::owner_bytes(uid, excluded_card_id, excluded_character_id);
if used + adding > MAX_BYTES_PER_USER {
return Err(format!(
"This upload would put your uploads at {} MB, over the {} MB per-account limit - delete a card or character first",
(used + adding) / (1024 * 1024), MAX_BYTES_PER_USER / (1024 * 1024)
));
}
Ok(())
}
// Edit a card in place. The master_card_id - and everything derived from it:
// master_skill_id, illust ids - stays the same, so a player who owns the card
// keeps owning the same card. master_character_id is fixed too: repointing it
@@ -1242,6 +1396,7 @@ pub fn update_card(uid: i64, master_card_id: i64, fields: &Fields) -> Result<(),
let mut card = build_card(master_card_id, master_character_id, fields, &stored)?;
let card_art = collect_card_art(fields, false)?;
card["art"] = merge_art(&stored["art"], &card_art);
check_quota(owner, database::card_bytes(&card), master_card_id, 0)?;
let lock = lock_onto_mutex!(UPLOAD_LOCK);
write_art(&card_dir(master_card_id), &card_art)?;
@@ -1313,10 +1468,12 @@ pub fn create_character(uid: i64, fields: &Fields) -> Result<i64, String> {
if !voice.is_empty() {
character["voice"] = voice;
}
check_quota(uid, database::character_bytes(&character), 0, 0)?;
write_art(&character_dir(master_character_id), &character_art)?;
write_voice(master_character_id, &voice_files)?;
database::insert_character(master_character_id, uid, &character);
database::insert_character(master_character_id, uid, &character)
.map_err(|e| format!("Could not store the character: {}", e))?;
database::bump_revision();
drop(lock);
@@ -1340,6 +1497,7 @@ pub fn update_character(uid: i64, master_character_id: i64, fields: &Fields) ->
if !voice.is_empty() {
character["voice"] = voice.clone();
}
check_quota(owner, database::character_bytes(&character), 0, master_character_id)?;
let lock = lock_onto_mutex!(UPLOAD_LOCK);
write_art(&character_dir(master_character_id), &character_art)?;
@@ -1376,6 +1534,44 @@ pub fn delete_character(uid: i64, master_character_id: i64) -> Result<(), String
Ok(())
}
// Every card this account uploaded, gone - called from userdata::delete_account,
// so a purged uploader leaves no catalog row resolving an owner id that no longer
// exists (browse renders an uploader name for every row). Player copies of the
// dead cards are wiped lazily on each account's next userdata pull, exactly like
// an owner-initiated delete.
//
// Characters go with them only when nothing else references them: a custom
// character that still backs SOMEONE ELSE'S card cannot be deleted without
// serving a dangling master_character_id, which the client throws on. Those stay,
// ownerless, which is the same trade delete_character already makes
pub fn purge_owner(uid: i64) {
if uid <= 0 {
return;
}
if disabled() {
return;
}
let cards = database::card_ids_for_owner(uid);
let characters = database::character_ids_for_owner(uid);
if cards.is_empty() && characters.is_empty() {
return;
}
let lock = lock_onto_mutex!(UPLOAD_LOCK);
for master_card_id in cards {
database::delete_card(master_card_id);
let _ = fs::remove_dir_all(card_dir(master_card_id));
}
for master_character_id in characters {
if database::cards_using_character(master_character_id) > 0 {
continue;
}
database::delete_character(master_character_id);
let _ = fs::remove_dir_all(character_dir(master_character_id));
}
database::bump_revision();
drop(lock);
}
async fn create(req: HttpRequest, payload: Multipart) -> HttpResponse {
if disabled() {
return HttpResponse::NotFound().finish();
@@ -1387,12 +1583,16 @@ async fn create(req: HttpRequest, payload: Multipart) -> HttpResponse {
Ok(fields) => fields,
Err(e) => return webui::error(&e)
};
match create_card(uid, &fields) {
Ok(master_card_id) => send_json(object!{
// Deriving 14 Lanczos3 PNGs up to 2048x1260 (and transcoding voicelines) is
// seconds of CPU: it belongs on the blocking pool, not on the actix worker
// that also has to keep serving the game API
match web::block(move || create_card(uid, &fields)).await {
Ok(Ok(master_card_id)) => send_json(object!{
result: "OK",
master_card_id: master_card_id
}),
Err(e) => webui::error(&e)
Ok(Err(e)) => webui::error(&e),
Err(_) => webui::error("The upload could not be processed")
}
}
@@ -1408,12 +1608,16 @@ async fn update(req: HttpRequest, payload: Multipart) -> HttpResponse {
Err(e) => return webui::error(&e)
};
let master_card_id = field_str(&fields, "master_card_id").parse::<i64>().unwrap_or(0);
match update_card(uid, master_card_id, &fields) {
Ok(()) => send_json(object!{
// Deriving 14 Lanczos3 PNGs up to 2048x1260 (and transcoding voicelines) is
// seconds of CPU: it belongs on the blocking pool, not on the actix worker
// that also has to keep serving the game API
match web::block(move || update_card(uid, master_card_id, &fields)).await {
Ok(Ok(())) => send_json(object!{
result: "OK",
master_card_id: master_card_id
}),
Err(e) => webui::error(&e)
Ok(Err(e)) => webui::error(&e),
Err(_) => webui::error("The edit could not be processed")
}
}
@@ -1461,12 +1665,16 @@ async fn character_create(req: HttpRequest, payload: Multipart) -> HttpResponse
Ok(fields) => fields,
Err(e) => return webui::error(&e)
};
match create_character(uid, &fields) {
Ok(master_character_id) => send_json(object!{
// Deriving 14 Lanczos3 PNGs up to 2048x1260 (and transcoding voicelines) is
// seconds of CPU: it belongs on the blocking pool, not on the actix worker
// that also has to keep serving the game API
match web::block(move || create_character(uid, &fields)).await {
Ok(Ok(master_character_id)) => send_json(object!{
result: "OK",
master_character_id: master_character_id
}),
Err(e) => webui::error(&e)
Ok(Err(e)) => webui::error(&e),
Err(_) => webui::error("The upload could not be processed")
}
}
@@ -1482,12 +1690,16 @@ async fn character_update(req: HttpRequest, payload: Multipart) -> HttpResponse
Err(e) => return webui::error(&e)
};
let master_character_id = field_str(&fields, "master_character_id").parse::<i64>().unwrap_or(0);
match update_character(uid, master_character_id, &fields) {
Ok(()) => send_json(object!{
// Deriving 14 Lanczos3 PNGs up to 2048x1260 (and transcoding voicelines) is
// seconds of CPU: it belongs on the blocking pool, not on the actix worker
// that also has to keep serving the game API
match web::block(move || update_character(uid, master_character_id, &fields)).await {
Ok(Ok(())) => send_json(object!{
result: "OK",
master_character_id: master_character_id
}),
Err(e) => webui::error(&e)
Ok(Err(e)) => webui::error(&e),
Err(_) => webui::error("The edit could not be processed")
}
}
@@ -1813,7 +2025,7 @@ pub mod tests {
let mut edit = Fields::new();
edit.insert(String::from("voice_result_bond_1"), test_wav(31.0, 6));
let err = with_permissions(4010, &[permissions::CARD_UPLOAD], || update_character(4010, id, &edit)).unwrap_err();
assert!(err.contains("voice_result_bond_1") && err.contains("maximum is 30"), "{}", err);
assert!(err.contains("voice_result_bond_1") && err.contains("30 second maximum"), "{}", err);
let mut edit = Fields::new();
edit.insert(String::from("voice_result_bond_1"), b"definitely not audio".to_vec());
assert!(with_permissions(4010, &[permissions::CARD_UPLOAD], || update_character(4010, id, &edit))
@@ -1823,6 +2035,145 @@ pub mod tests {
wipe(4010);
}
// The home-screen moments ride the exact same upload path as the live/skill
// ones: real create + edit, transcode, content addressing, renumbering.
// day_1225 is the interesting one - the client turns it into a row carrying
// a date_condition id rather than 0
#[test]
fn home_voicelines_upload_through_the_real_path() {
let _lock = crate::runtime::lock_test_data_path();
wipe(4011);
let mut fields = character_fields();
fields.insert(String::from("voice_random_1"), test_wav(1.0, 11));
field(&mut fields, "voice_random_1_text", "おはよう!");
field(&mut fields, "voice_random_1_text_en", "Morning!");
fields.insert(String::from("voice_random_3"), test_wav(1.0, 12));
fields.insert(String::from("voice_touch_1"), test_wav(0.5, 13));
fields.insert(String::from("voice_time_morning_1"), test_wav(0.5, 14));
fields.insert(String::from("voice_day_1225_1"), test_wav(0.5, 15));
fields.insert(String::from("voice_trigger_friend_1"), test_wav(0.5, 16));
// Still ignored: a real moment, an index past the variant cap
fields.insert(String::from("voice_random_10"), test_wav(1.0, 17));
let id = with_permissions(4011, &[permissions::CARD_UPLOAD], || create_character(4011, &fields).unwrap());
let character = database::get_character(id).unwrap();
let voice = &character["voice"];
assert_eq!(voice.len(), 6);
let kinds: Vec<String> = voice.members().map(|line| line["kind"].to_string()).collect();
for kind in ["random", "touch", "time_morning", "day_1225", "trigger_friend"] {
assert!(kinds.contains(&String::from(kind)), "{} missing from {:?}", kind, kinds);
}
// The sparse random indexes renumbered to 1 and 2, captions intact
let random: Vec<&JsonValue> = voice.members().filter(|line| line["kind"] == "random").collect();
assert_eq!(random.len(), 2);
assert_eq!(random[0]["index"].as_i64(), Some(1));
assert_eq!(random[1]["index"].as_i64(), Some(2));
assert_eq!(random[0]["text"].as_str(), Some("おはよう!"));
assert_eq!(random[0]["text_en"].as_str(), Some("Morning!"));
for line in voice.members() {
let md5 = line["md5"].to_string();
let bytes = fs::read(voice_path(id, &md5)).unwrap();
assert!(bytes.starts_with(b"OggS"), "transcoded to ogg");
assert_eq!(database::find_voice_by_md5(&md5), Some(format!("characters/{}/voice/{}.ogg", id, md5)));
}
// Editing one home moment leaves the others alone
let touch_md5 = voice.members().find(|line| line["kind"] == "touch").unwrap()["md5"].to_string();
let mut edit = Fields::new();
field(&mut edit, "voice_day_1225_1_text", "メリークリスマス!");
edit.insert(String::from("voice_touch_2"), test_wav(0.5, 18));
with_permissions(4011, &[permissions::CARD_UPLOAD], || update_character(4011, id, &edit).unwrap());
let after = database::get_character(id).unwrap();
assert_eq!(after["voice"].len(), 7);
assert_eq!(after["voice"].members().find(|line| line["kind"] == "day_1225").unwrap()["text"].as_str(),
Some("メリークリスマス!"));
assert_eq!(after["voice"].members()
.find(|line| line["kind"] == "touch" && line["index"] == 1).unwrap()["md5"].to_string(), touch_md5);
// The 30-second and rich-text rules apply to the new moments too
let mut edit = Fields::new();
edit.insert(String::from("voice_time_night_1"), test_wav(31.0, 19));
let err = with_permissions(4011, &[permissions::CARD_UPLOAD], || update_character(4011, id, &edit)).unwrap_err();
assert!(err.contains("voice_time_night_1") && err.contains("30 second maximum"), "{}", err);
let mut edit = Fields::new();
field(&mut edit, "voice_touch_1_text", "<color=red>x");
assert!(with_permissions(4011, &[permissions::CARD_UPLOAD], || update_character(4011, id, &edit))
.unwrap_err().contains("<color>"));
wipe(4011);
}
// A misspelled moment must not upload as silence-that-never-plays
#[test]
fn an_unknown_voice_kind_is_rejected() {
let _lock = crate::runtime::lock_test_data_path();
wipe(4012);
let reject = |key: &str| {
let mut fields = character_fields();
fields.insert(String::from(key), test_wav(0.5, 21));
let err = with_permissions(4012, &[permissions::CARD_UPLOAD], || create_character(4012, &fields)).unwrap_err();
assert!(err.contains(key) && err.contains("not a voiceline field"), "{}: {}", key, err);
};
reject("voice_time_mornng_1");
reject("voice_day_0102_1");
reject("voice_live_start");
reject("voice_random_x");
// Captions and delete flags go through the same check
let mut fields = character_fields();
field(&mut fields, "voice_seaon_spring_1_text", "x");
let err = with_permissions(4012, &[permissions::CARD_UPLOAD], || create_character(4012, &fields)).unwrap_err();
assert!(err.contains("voice_seaon_spring_1_text"), "{}", err);
// ...and a correctly named one still goes through
let mut fields = character_fields();
fields.insert(String::from("voice_season_spring_1"), test_wav(0.5, 22));
let id = with_permissions(4012, &[permissions::CARD_UPLOAD], || create_character(4012, &fields).unwrap());
assert_eq!(database::get_character(id).unwrap()["voice"][0]["kind"].as_str(), Some("season_spring"));
wipe(4012);
}
// The webui renders whatever this serves; a kind that only exists in one of
// the two lists is the bug this endpoint exists to prevent
#[test]
fn the_limits_endpoint_lists_the_voice_kinds() {
let limits = upload_limits();
assert_eq!(limits["max_voice_variants"].as_usize(), Some(MAX_VOICE_VARIANTS));
assert_eq!(limits["max_voice_bytes"].as_usize(), Some(MAX_VOICE_BYTES));
assert_eq!(limits["max_voice_seconds"].as_f64(), Some(MAX_VOICE_SECONDS));
let served = &limits["voice_kinds"];
assert_eq!(served.len(), VOICE_KINDS.len());
for (entry, kind) in served.members().zip(VOICE_KINDS) {
assert_eq!(entry["name"].as_str(), Some(kind.name));
assert_eq!(entry["label"].as_str(), Some(kind.label));
assert_eq!(entry["label_en"].as_str(), Some(kind.label_en));
assert_eq!(entry["category"].as_str(), Some(kind.category));
assert!(!kind.label.is_empty() && !kind.label_en.is_empty(), "{} needs both labels", kind.name);
assert!(["home", "live", "result", "skill"].contains(&kind.category), "{}: {}", kind.name, kind.category);
}
// Names are the multipart infix and the client's switch arm: unique,
// snake_case, and free of the separators the field parser splits on
let mut names: Vec<&str> = VOICE_KINDS.iter().map(|kind| kind.name).collect();
names.sort_unstable();
let count = names.len();
names.dedup();
assert_eq!(names.len(), count, "duplicate voice kind name");
for kind in VOICE_KINDS {
assert!(kind.name.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_'), "{}", kind.name);
// The field parser strips these suffixes before splitting off the index
assert!(!kind.name.ends_with("_text") && !kind.name.ends_with("_text_en") && !kind.name.ends_with("_delete"), "{}", kind.name);
// ...and splits at the LAST underscore, so "{another kind}_{number}"
// would be two readings of the same field name
if let Some((head, tail)) = kind.name.rsplit_once('_') {
assert!(tail.parse::<usize>().is_err() || !VOICE_KINDS.iter().any(|other| other.name == head),
"{} is ambiguous with {}", kind.name, head);
}
}
}
// Diagnostic + sanity guard for the derived skill magnitude envelopes
#[test]
fn skill_ranges_derive_from_shipped_rows() {
@@ -2274,4 +2625,106 @@ pub mod tests {
assert!(!viewer_can_resolve(100_010_001, 1));
assert!(viewer_can_resolve(100_010_001, 2));
}
// ---- defect-fix coverage -------------------------------------------------
// D14: card art lives at fixed per-card filenames that an in-place edit
// overwrites, so the md5 index can briefly point at bytes that are no longer
// its own. The client caches by md5 and never re-checks, so the route verifies
// the bytes before it serves them
#[test]
fn the_data_route_refuses_bytes_that_do_not_match_the_md5() {
let _lock = crate::runtime::lock_test_data_path();
wipe(4090);
let png = seeded_png(120, 60, 91);
let md5 = format!("{:x}", md5::compute(&png));
let id = database::next_card_id();
database::insert_card(id, 1001, 4090, &object!{
"master_card_id": id,
"rarity": 1,
"art": [{ "kind": "c", "variant": "00", "md5": md5.clone(), "size": png.len() }]
}, true, true).unwrap();
fs::create_dir_all(card_dir(id)).unwrap();
let file = format!("{}/c_00.png", card_dir(id));
fs::write(&file, &png).unwrap();
let call = || -> actix_web::http::StatusCode {
let req = actix_web::test::TestRequest::default()
.param("hash", md5.clone())
.param("file", format!("{}.png", md5))
.to_http_request();
actix_web::rt::System::new().block_on(async { data(req).await }).status()
};
assert_eq!(call(), actix_web::http::StatusCode::OK);
// The index still resolves, but the file no longer holds those bytes
fs::write(&file, seeded_png(120, 60, 92)).unwrap();
assert_eq!(call(), actix_web::http::StatusCode::NOT_FOUND);
fs::write(&file, &png).unwrap();
assert_eq!(call(), actix_web::http::StatusCode::OK);
wipe(4090);
}
// D13: JsonValue::to_string renders Null as the literal "null", so a stored
// line with no md5 used to aim the unlink at a file called null.ogg
#[test]
fn the_voice_gc_ignores_a_missing_md5() {
let _lock = crate::runtime::lock_test_data_path();
let character = 5_900_001;
let dir = format!("{}/voice", character_dir(character));
fs::create_dir_all(&dir).unwrap();
let decoy = format!("{}/null.ogg", dir);
fs::write(&decoy, b"not a voiceline").unwrap();
let real_md5 = "b1".repeat(16);
let real = voice_path(character, &real_md5);
fs::write(&real, b"a voiceline").unwrap();
let old = array![
{ "kind": "live_start", "index": 1, "md5": JsonValue::Null },
{ "kind": "live_start", "index": 2, "md5": real_md5.clone() }
];
gc_voice(character, &old, &array![]);
assert!(fs::read(&decoy).is_ok(), "the GC unlinked a file named after JSON null");
assert!(fs::read(&real).is_err(), "the dropped line's ogg was kept");
let _ = fs::remove_dir_all(character_dir(character));
}
// D15: cards and characters share one storage root, so they share one
// per-account byte quota
#[test]
fn uploads_are_bounded_by_a_per_account_byte_quota() {
let _lock = crate::runtime::lock_test_data_path();
wipe(4091);
assert_eq!(database::owner_bytes(4091, 0, 0), 0);
let id = database::next_card_id();
database::insert_card(id, 1001, 4091, &object!{
"master_card_id": id,
"rarity": 1,
"art": [{ "kind": "c", "variant": "00", "md5": "a1".repeat(16), "size": 1000 }]
}, true, true).unwrap();
let character = database::next_character_id();
database::insert_character(character, 4091, &object!{
"master_character_id": character,
"name": "Quota",
"art": [{ "kind": "icon", "md5": "a2".repeat(16), "size": 500 }],
"voice": [{ "kind": "live_start", "index": 1, "md5": "a3".repeat(16), "size": 250 }]
}).unwrap();
assert_eq!(database::owner_bytes(4091, 0, 0), 1750);
// An in-place edit replaces its own bytes rather than adding to them
assert_eq!(database::owner_bytes(4091, id, 0), 750);
assert_eq!(database::owner_bytes(4091, 0, character), 1000);
// Another account's uploads are not on this one's bill
assert_eq!(database::owner_bytes(4092, 0, 0), 0);
assert!(check_quota(4091, 1, 0, 0).is_ok());
assert!(check_quota(4091, MAX_BYTES_PER_USER, 0, 0).unwrap_err().contains("per-account limit"));
wipe(4091);
}
}
+779 -33
View File
File diff suppressed because it is too large Load Diff
+65 -7
View File
@@ -7,7 +7,7 @@ use symphonia::core::formats::probe::Hint;
use symphonia::core::io::MediaSourceStream;
use vorbis_rs::{VorbisBitrateManagementStrategy, VorbisEncoderBuilder};
use super::{DEFAULT_PREVIEW_LENGTH_SEC, PREVIEW_FADE_SEC};
use super::{DEFAULT_PREVIEW_LENGTH_SEC, MAX_AUDIO_SECONDS, PREVIEW_FADE_SEC};
// The whole audio pipeline runs in-process: symphonia (pure Rust) decodes and
// validates uploads, vorbis_rs (libvorbis compiled into the binary - a library
@@ -53,7 +53,12 @@ fn is_ogg_vorbis(bytes: &[u8]) -> bool {
bytes.starts_with(b"OggS") && bytes.len() > 64 && bytes[..64].windows(7).any(|w| w == b"\x01vorbis")
}
fn decode(bytes: &[u8]) -> Result<DecodedAudio, String> {
// `max_duration_sec` is enforced INSIDE the packet loop, not after it: the decode
// accumulates full planar f32 PCM, so checking the duration afterwards means the
// allocation has already happened (an hour of 44.1kHz stereo is ~1.4GB of Vec).
// Bailing at the first packet past the limit keeps the peak proportional to the
// limit instead of to the upload
fn decode(bytes: &[u8], max_duration_sec: f64) -> Result<DecodedAudio, String> {
let stream = MediaSourceStream::new(Box::new(std::io::Cursor::new(bytes.to_vec())), Default::default());
let mut format = symphonia::default::get_probe()
.probe(&Hint::new(), stream, Default::default(), Default::default())
@@ -96,6 +101,9 @@ fn decode(bytes: &[u8]) -> Result<DecodedAudio, String> {
for (i, samples) in channels.iter_mut().enumerate() {
samples.extend(interleaved.iter().skip(i).step_by(count));
}
if sample_rate > 0 && channels[0].len() as f64 / sample_rate as f64 > max_duration_sec {
return Err(format!("Audio is over the {:.0} second maximum", max_duration_sec));
}
}
if channels.is_empty() || channels[0].is_empty() || sample_rate == 0 {
@@ -141,14 +149,13 @@ fn cue(bytes: Vec<u8>, duration_sec: f64) -> Cue {
// reads, keep it as-is when it's already ogg-vorbis, otherwise transcode. No
// cuts, fades, loop points or preview split - mono or stereo as-sourced
pub fn process_one_shot(bytes: &[u8], max_duration_sec: f64) -> Result<Cue, String> {
let audio = decode(bytes)?;
// The length limit belongs to the decoder, which stops at the first packet
// past it rather than accumulating the whole clip and rejecting it afterwards
let audio = decode(bytes, max_duration_sec)?;
let duration = audio.duration();
if duration < 0.2 {
return Err(String::from("Audio clip is shorter than 0.2 seconds"));
}
if duration > max_duration_sec {
return Err(format!("Audio clip is {:.1} seconds long - the maximum is {:.0} seconds", duration, max_duration_sec));
}
if is_ogg_vorbis(bytes) {
return Ok(cue(bytes.to_vec(), duration));
}
@@ -160,7 +167,7 @@ pub fn process_one_shot(bytes: &[u8], max_duration_sec: f64) -> Result<Cue, Stri
// fades. Both are stored content-addressed by the md5 of the final ogg bytes -
// the client validates md5(file) against the value served in the catalog
pub fn process(bytes: &[u8], preview_start_sec: Option<f64>, preview_length_sec: Option<f64>) -> Result<(Cue, Cue), String> {
let audio = decode(bytes)?;
let audio = decode(bytes, MAX_AUDIO_SECONDS)?;
let duration = audio.duration();
if duration <= 1.0 {
return Err(String::from("Audio track is too short"));
@@ -200,3 +207,54 @@ pub fn process(bytes: &[u8], preview_start_sec: Option<f64>, preview_length_sec:
Ok((play, select))
}
#[cfg(test)]
mod tests {
use super::*;
// 44.1kHz 16-bit mono, `seconds` long
fn wav(seconds: f64) -> Vec<u8> {
let sample_rate: u32 = 44100;
let frames = (seconds * sample_rate as f64) as u32;
let data_len = frames * 2;
let mut rv = Vec::new();
rv.extend(b"RIFF");
rv.extend((36 + data_len).to_le_bytes());
rv.extend(b"WAVEfmt ");
rv.extend(16u32.to_le_bytes());
rv.extend(1u16.to_le_bytes());
rv.extend(1u16.to_le_bytes());
rv.extend(sample_rate.to_le_bytes());
rv.extend((sample_rate * 2).to_le_bytes());
rv.extend(2u16.to_le_bytes());
rv.extend(16u16.to_le_bytes());
rv.extend(b"data");
rv.extend(data_len.to_le_bytes());
for i in 0..frames {
let sample = ((i as f64 * 440.0 * 2.0 * std::f64::consts::PI / sample_rate as f64).sin() * 8000.0) as i16;
rv.extend(sample.to_le_bytes());
}
rv
}
// The length limit is enforced from INSIDE the packet loop: the decode
// accumulates full planar f32 PCM, so a post-decode check means the
// allocation already happened (an hour of stereo is ~1.4GB of Vec)
#[test]
fn the_decoder_stops_at_the_duration_cap() {
let three_seconds = wav(3.0);
let audio = decode(&three_seconds, 5.0).unwrap();
assert!((audio.duration() - 3.0).abs() < 0.01);
let Err(err) = decode(&three_seconds, 1.0) else {
panic!("a three-second track decoded under a one-second cap");
};
assert!(err.contains("1 second maximum"), "{}", err);
// The same cap is what refuses an over-long voiceline
let Err(err) = process_one_shot(&three_seconds, 1.0) else {
panic!("an over-long one-shot was accepted");
};
assert!(err.contains("1 second maximum"), "{}", err);
assert!(process_one_shot(&three_seconds, 5.0).is_ok());
}
}
+33 -5
View File
@@ -1,4 +1,5 @@
use jzon::{object, JsonValue};
use std::collections::HashMap;
// Transcodes a SIF1/NPPS4 beatmap (array of {timing_sec, effect, effect_value, position})
// into the SIF2 chart JSON the client deserializes into NoteData.
@@ -87,6 +88,14 @@ fn simultaneous(a: f64, b: f64) -> bool {
const MISS_OFFSET_SEC: f64 = 0.15;
const MISS_OFFSET_SLIDER_SEC: f64 = 0.34;
// The most notes one difficulty may carry. Every transcode step is linear in this
// and the chart JSON is bounded only in bytes by the upload caps, so an explicit
// ceiling is what keeps a 64MB chart from turning into tens of millions of
// JsonValue allocations on a worker. The hardest shipped SIF2 chart is under 1500
// notes and the whole 2146-chart official set peaks well below that, so this is
// more than an order of magnitude of headroom
pub const MAX_NOTES: usize = 20_000;
// MarkerData.IsSliderMarker: a chained note with a cross-lane parent or child.
fn is_slider(data: &JsonValue, line_of: &dyn Fn(i64) -> Option<i64>) -> bool {
let parent_id = data["parent_id"].as_i64().unwrap_or(0);
@@ -106,10 +115,12 @@ fn is_slider(data: &JsonValue, line_of: &dyn Fn(i64) -> Option<i64>) -> bool {
// m_MusicDuration is LiveMst._endWait + the music length, and _endWait is 0 in every one of the
// 637 official live rows and in ours), so this is what has to fit inside the audio.
pub fn end_time(chart: &JsonValue) -> f64 {
let lines: Vec<(i64, i64)> = chart["notes"].members().skip(1)
// Indexed, not scanned: this runs once per note over a chart whose note count
// is only bounded by MAX_NOTES, and the linear lookup made it quadratic
let lines: HashMap<i64, i64> = chart["notes"].members().skip(1)
.map(|n| (n["id"].as_i64().unwrap_or(0), n["line"].as_i64().unwrap_or(0)))
.collect();
let line_of = |id: i64| lines.iter().find(|(i, _)| *i == id).map(|(_, line)| *line);
let line_of = |id: i64| lines.get(&id).copied();
let mut end: f64 = 0.0;
for data in chart["notes"].members().skip(1) {
@@ -171,6 +182,12 @@ fn parse_sif_note(data: &JsonValue, index: usize) -> Result<(f64, i64, f64, i64,
if !(1..=9).contains(&position) {
return Err(format!("Note {}: position {} is outside 1-9", index, position));
}
// NaN and the infinities pass every comparison below and then reach the
// time-order sort, whose partial_cmp().unwrap() panics on an incomparable
// pair - a worker panic authored by the uploaded file
if !timing.is_finite() || !effect_value.is_finite() {
return Err(format!("Note {}: timing_sec/effect_value must be finite numbers", index));
}
if timing < 0.0 {
return Err(format!("Note {}: negative timing_sec {}", index, timing));
}
@@ -186,17 +203,28 @@ pub fn transcode(beatmap: &JsonValue) -> Result<(JsonValue, i64), String> {
if !beatmap.is_array() || beatmap.is_empty() {
return Err(String::from("Chart is not a JSON array of notes"));
}
if beatmap.len() > MAX_NOTES {
return Err(format!("Chart has {} notes - the maximum is {}", beatmap.len(), MAX_NOTES));
}
let mut work: Vec<WorkNote> = Vec::new();
// Slide chain id -> the work indices in that chain, in input order
let mut chains: Vec<(i64, Vec<usize>)> = Vec::new();
// (timing bits, position) -> effect, for the duplicate check below. Indexed
// rather than rescanned: the old scan-all-preceding-notes loop was quadratic
// over an input whose size the upload caps only bound in bytes. The key uses
// the raw f64 bits, which is exactly the equality the scan tested (both
// infinities and NaN are already rejected in parse_sif_note, so bit equality
// and value equality agree here)
let mut seen: HashMap<(u64, i64), i64> = HashMap::new();
for (i, data) in beatmap.members().enumerate() {
let (timing, effect, effect_value, position, group) = parse_sif_note(data, i)?;
for other in beatmap.members().take(i) {
if other["timing_sec"].as_f64() == Some(timing) && other["position"].as_i64() == Some(position) && other["effect"].as_i64() != Some(effect) {
match seen.insert((timing.to_bits(), position), effect) {
Some(other) if other != effect => {
return Err(format!("Note {}: duplicate timing {} on position {} with a different effect", i, timing, position));
}
},
_ => {}
}
let head = work.len();
+39 -8
View File
@@ -41,11 +41,40 @@ pub fn build(music_id: i64) -> Result<Vec<u8>, String> {
Ok(zip.finish().map_err(|e| e.to_string())?.into_inner())
}
fn read_entry<R: Read + Seek>(archive: &mut zip::ZipArchive<R>, name: &str) -> Option<Vec<u8>> {
let mut file = archive.by_name(name).ok()?;
// Reads one entry, capped. Deflate's ceiling is about 1032:1, so an uncapped
// read_to_end here is a zip bomb: a one-megabyte entry inflates to a gigabyte and
// grows the Vec until the allocator or the OOM killer stops it. Every entry is
// bounded by the upload form's own per-file cap, and by what is left of the
// per-request budget across all entries.
//
// The central directory's declared size rejects the obvious case without
// inflating anything; take(cap + 1) makes that declaration untrusted - a lying
// header runs out of budget one byte past the cap and stops there.
//
// Ok(None) is "the package does not carry this entry", which is a normal outcome
// for the optional ones
fn read_entry<R: Read + Seek>(archive: &mut zip::ZipArchive<R>, name: &str, remaining: &mut usize) -> Result<Option<Vec<u8>>, String> {
let Ok(mut file) = archive.by_name(name) else {
return Ok(None);
};
let cap = std::cmp::min(super::MAX_FILE_BYTES, *remaining);
// Which limit the entry actually ran into, so the message names the right one
let too_big = if cap >= super::MAX_FILE_BYTES {
super::over_file_limit(name)
} else {
super::over_request_limit()
};
if file.size() > cap as u64 {
return Err(too_big);
}
let mut bytes = Vec::new();
file.read_to_end(&mut bytes).ok()?;
Some(bytes)
file.by_ref().take(cap as u64 + 1).read_to_end(&mut bytes)
.map_err(|_| format!("Package entry '{}' could not be read", name))?;
if bytes.len() > cap {
return Err(too_big);
}
*remaining -= bytes.len();
Ok(Some(bytes))
}
// Expands a package into the same field map the upload form produces - the zip
@@ -55,8 +84,10 @@ fn read_entry<R: Read + Seek>(archive: &mut zip::ZipArchive<R>, name: &str) -> O
// visibility/shared_with/downloads_disabled aren't packaged and stay untouched
pub fn expand(package: &[u8], fields: &mut HashMap<String, Vec<u8>>) -> Result<(), String> {
let mut archive = zip::ZipArchive::new(Cursor::new(package)).map_err(|_| String::from("Package is not a valid zip file"))?;
// The decompressed budget for the whole package, shared by every entry
let mut remaining = super::MAX_REQUEST_BYTES;
let manifest = read_entry(&mut archive, "manifest.json").ok_or(String::from("Package has no manifest.json"))?;
let manifest = read_entry(&mut archive, "manifest.json", &mut remaining)?.ok_or(String::from("Package has no manifest.json"))?;
let manifest = jzon::parse(&String::from_utf8_lossy(&manifest)).map_err(|_| String::from("Package manifest is not valid JSON"))?;
if manifest["format"].as_i64() != Some(1) {
return Err(String::from("Unsupported package format"));
@@ -74,11 +105,11 @@ pub fn expand(package: &[u8], fields: &mut HashMap<String, Vec<u8>>) -> Result<(
}
}
fields.insert(String::from("jacket"), read_entry(&mut archive, "jacket").ok_or(String::from("Package has no jacket"))?);
fields.insert(String::from("audio"), read_entry(&mut archive, "audio").ok_or(String::from("Package has no audio"))?);
fields.insert(String::from("jacket"), read_entry(&mut archive, "jacket", &mut remaining)?.ok_or(String::from("Package has no jacket"))?);
fields.insert(String::from("audio"), read_entry(&mut archive, "audio", &mut remaining)?.ok_or(String::from("Package has no audio"))?);
let mut has_chart = false;
for level in 1..=LEVEL_COUNT {
if let Some(chart) = read_entry(&mut archive, &format!("chart_{}.json", level)) {
if let Some(chart) = read_entry(&mut archive, &format!("chart_{}.json", level), &mut remaining)? {
fields.insert(format!("chart_{}", level), chart);
has_chart = true;
}
+34 -2
View File
@@ -50,6 +50,10 @@ static ASSET_VERSIONS: &[AssetVersion] = &[
AssetVersion { region: "JP", platform: "Android", version: "01a71b00f63e4dba92117ac7e60070a6", hash: "d12cecc5695da7f81f8873a3ff93752e", latest: true },
AssetVersion { region: "JP", platform: "iOS", version: "01a71b00f63e4dba92117ac7e60070a6", hash: "7c1f61ee68ac84c82dd397a162629142", latest: true },
AssetVersion { region: "JP", platform: "Linux", version: "01a71b00f63e4dba92117ac7e60070a6", hash: "", latest: true },
AssetVersion { region: "JP", platform: "macOS", version: "01a71b00f63e4dba92117ac7e60070a6", hash: "", latest: true },
//AssetVersion { region: "JP", platform: "WebGL", version: "4c921d2443335e574a82e04ec9ea243c", hash: "e1ff7c74b20c8d216507972b6f24b9df", latest: true },
];
@@ -68,6 +72,8 @@ impl AssetVersion {
let ov = args.asset_version.as_str();
let oh = match (self.region, self.platform) {
("JP", "Windows") => args.windows_asset_hash.as_str(),
("JP", "Linux") => args.linux_asset_hash.as_str(),
("JP", "macOS") => args.macos_asset_hash.as_str(),
("JP", "Android") => args.jp_android_asset_hash.as_str(),
("JP", "iOS") => args.jp_ios_asset_hash.as_str(),
("GL", "Android") => args.en_android_asset_hash.as_str(),
@@ -89,7 +95,7 @@ fn valid_hashes(asset_version: &str, platform: &str) -> Vec<String> {
if entry.platform != platform {
continue;
}
if entry.version == asset_version {
if entry.version == asset_version && !entry.hash.is_empty() {
out.push(entry.stock_hash());
}
if entry.latest {
@@ -116,7 +122,7 @@ fn preferred_hash(asset_version: &str, platform: &str) -> Option<String> {
}
}
}
if entry.version == asset_version && stock.is_none() {
if entry.version == asset_version && stock.is_none() && !entry.hash.is_empty() {
stock = Some(entry.stock_hash());
}
}
@@ -169,6 +175,8 @@ pub fn parse_platform(header: &str) -> &str {
"iphone" => "iOS",
"windows" => "Windows",
"windowsplayer" => "Windows",
"linuxplayer" => "Linux",
"osxplayer" => "macOS",
"webglplayer" => "WebGL",
"editor" => "Editor",
"windowseditor" => "Editor",
@@ -471,3 +479,27 @@ pub(crate) fn get_cards(arr: JsonValue, user: &JsonValue) -> JsonValue {
}
rv
}
#[cfg(test)]
mod platform_tests {
use super::*;
#[test]
fn standalone_player_platforms_are_known() {
assert_eq!(parse_platform("WindowsPlayer"), "Windows");
assert_eq!(parse_platform("LinuxPlayer"), "Linux");
assert_eq!(parse_platform("OSXPlayer"), "macOS");
assert_eq!(parse_platform("OSXEditor"), "Editor");
}
// Linux / macOS have no baked hash: without the CLI override the platform has no valid
// hash at all (never an empty string presented as one)
#[test]
fn unhashed_platforms_answer_nothing_without_an_override() {
let latest = ASSET_VERSIONS.iter().find(|e| e.latest && e.platform == "Windows").unwrap().version;
assert_eq!(preferred_hash(latest, "macOS"), None);
assert_eq!(preferred_hash(latest, "Linux"), None);
assert!(valid_hashes(latest, "macOS").is_empty());
assert!(preferred_hash(latest, "Windows").is_some());
}
}
+6 -6
View File
@@ -427,7 +427,7 @@ async fn lottery_post(req: HttpRequest, Session { key, body }: Session) -> impl
}
// tests!!!
#[cfg(test)]
mod tests {
@@ -447,18 +447,18 @@ mod tests {
let mut r1_ids = Vec::new();
for seed in 0..3 {
let id = custom_card_db::next_card_id();
custom_card_db::insert_card(id, 1001, 6001, &object!{ "master_card_id": id, "rarity": 1, "seed": seed }, true, true);
custom_card_db::insert_card(id, 1001, 6001, &object!{ "master_card_id": id, "rarity": 1, "seed": seed }, true, true).unwrap();
r1_ids.push(id);
}
let r2 = custom_card_db::next_card_id();
custom_card_db::insert_card(r2, 1001, 6001, &object!{ "master_card_id": r2, "rarity": 2 }, true, true);
custom_card_db::insert_card(r2, 1001, 6001, &object!{ "master_card_id": r2, "rarity": 2 }, true, true).unwrap();
let r3 = custom_card_db::next_card_id();
custom_card_db::insert_card(r3, 1001, 6001, &object!{ "master_card_id": r3, "rarity": 3 }, true, true);
custom_card_db::insert_card(r3, 1001, 6001, &object!{ "master_card_id": r3, "rarity": 3 }, true, true).unwrap();
// Draft / unobtainable cards must never come out of the pool
let draft = custom_card_db::next_card_id();
custom_card_db::insert_card(draft, 1001, 6001, &object!{ "master_card_id": draft, "rarity": 1 }, false, true);
custom_card_db::insert_card(draft, 1001, 6001, &object!{ "master_card_id": draft, "rarity": 1 }, false, true).unwrap();
let unobtainable = custom_card_db::next_card_id();
custom_card_db::insert_card(unobtainable, 1001, 6001, &object!{ "master_card_id": unobtainable, "rarity": 1 }, true, false);
custom_card_db::insert_card(unobtainable, 1001, 6001, &object!{ "master_card_id": unobtainable, "rarity": 1 }, true, false).unwrap();
let drawn = custom_banner_cards(11);
assert_eq!(drawn.len(), 11);
+7 -24
View File
@@ -71,8 +71,6 @@ pub enum UserView {
const DEFAULT_CARD: i64 = 10010001;
lazy_static! {
// Each character's lowest official card id: the stand-in shown to viewers
// who can't resolve a custom card of that character
static ref OFFICIAL_CARD_BY_CHARACTER: HashMap<i64, i64> = {
let mut rv: HashMap<i64, i64> = HashMap::new();
for entry in databases::CARD_LIST.entries() {
@@ -90,8 +88,6 @@ lazy_static! {
};
}
// The character behind any card id: baked masterdata first, then the runtime
// custom-card db, then the imported band's id arithmetic (prefix - 9000)
fn card_character(id: i64) -> Option<i64> {
let card = &databases::CARD_LIST[id.to_string()];
if !card.is_empty() {
@@ -103,9 +99,6 @@ fn card_character(id: i64) -> Option<i64> {
Some(id / 10000 - 9000)
}
// A custom card the viewer can't resolve shows as its character's base
// official card - the right face, never a crash. Characters with no official
// card (custom ones included) fall back to the default
fn proxy_card_id(id: i64) -> i64 {
if !card::is_custom(id) {
return id;
@@ -119,8 +112,6 @@ fn proxy_card_id(id: i64) -> i64 {
*rv
}
// A card row for a slot the account can't actually supply: level 1, unevolved.
// Only ever handed to viewers, never written back to the account
fn stand_in_card(master_card_id: i64) -> JsonValue {
object!{
id: master_card_id,
@@ -131,13 +122,6 @@ fn stand_in_card(master_card_id: i64) -> JsonValue {
}
}
// The card object for one of the four favourite/guest slots. global::get_card
// hands back an empty object when the slot is 0 (never set) or names a card the
// account no longer holds, and an empty object reaches the client as
// master_card_id 0: Shock.CardData's constructor looks that up in masterdata and
// dereferences the row, so it throws before the guest cell is ever drawn. Same
// repair userdata::remove_deleted_custom_cards makes for a dead slot - the
// account's first card, then the default for an account holding none
fn slot_card(id: i64, user: &JsonValue) -> JsonValue {
let card = global::get_card(id, user);
if !card.is_empty() {
@@ -213,10 +197,6 @@ pub fn get_user(id: i64, friends: &JsonValue, view: UserView, protocol: u32) ->
guest_pure_card: slot_card(user["user"]["guest_pure_master_card_id"].as_i64().unwrap_or(0), &user)
};
// The id fields have to name the same card as the objects: surfaces that
// resolve the id instead of the object (profile, friend detail) would
// otherwise look up the slot this just stood in for. A no-op for an account
// whose slots are all set
for (key, card) in [
("favorite_master_card_id", "favorite_card"),
("guest_smile_master_card_id", "guest_smile_card"),
@@ -268,6 +248,9 @@ pub fn get_user(id: i64, friends: &JsonValue, view: UserView, protocol: u32) ->
rv
}
// here are some tests that ai wrote...
#[cfg(test)]
mod tests {
use super::*;
@@ -326,11 +309,11 @@ mod tests {
// A runtime card on an official character proxies to that character
let id = db::next_card_id();
db::insert_card(id, 2003, 5101, &jzon::object!{ "master_card_id": id, "rarity": 1 }, true, false);
db::insert_card(id, 2003, 5101, &jzon::object!{ "master_card_id": id, "rarity": 1 }, true, false).unwrap();
assert_eq!(proxy_card_id(id), 20030001);
// On a custom character (no official card) it falls to the default
let orphan = db::next_card_id();
db::insert_card(orphan, db::FIRST_CHARACTER_ID, 5101, &jzon::object!{ "master_card_id": orphan, "rarity": 1 }, true, false);
db::insert_card(orphan, db::FIRST_CHARACTER_ID, 5101, &jzon::object!{ "master_card_id": orphan, "rarity": 1 }, true, false).unwrap();
assert_eq!(proxy_card_id(orphan), DEFAULT_CARD);
// A deleted/unknown runtime id can't resolve a character either
let unknown = db::next_card_id() + 5000;
@@ -353,9 +336,9 @@ mod tests {
wipe(5102);
let published = db::next_card_id();
db::insert_card(published, 2003, 5102, &jzon::object!{ "master_card_id": published, "rarity": 1 }, true, false);
db::insert_card(published, 2003, 5102, &jzon::object!{ "master_card_id": published, "rarity": 1 }, true, false).unwrap();
let draft = db::next_card_id();
db::insert_card(draft, 2003, 5102, &jzon::object!{ "master_card_id": draft, "rarity": 1 }, false, false);
db::insert_card(draft, 2003, 5102, &jzon::object!{ "master_card_id": draft, "rarity": 1 }, false, false).unwrap();
assert!(custom_card::viewer_can_resolve(published, custom_card::PROTOCOL_VERSION));
assert!(!custom_card::viewer_can_resolve(draft, custom_card::PROTOCOL_VERSION));
+20 -102
View File
@@ -97,54 +97,6 @@ INSERT OR IGNORE INTO exchange (user_id, exchange) SELECT user_id, '[]' FROM use
}
}
// maybe we will use this later
/*
pub fn downgrade_account_cards(user: &JsonValue) -> JsonValue {
let mut rv = user.clone();
let mut cards = array![];
let mut ids = array![];
for data in user["card_list"].members() {
let id = data["master_card_id"].as_i64().unwrap_or(0);
let downgraded = guest::proxy_card_id(id);
// Whole characters share one downgrade, and the client can't hold the
// same card twice
if ids.contains(downgraded) {
continue;
}
ids.push(downgraded).unwrap();
let mut data = data.clone();
if downgraded != id {
data["id"] = downgraded.into();
data["master_card_id"] = downgraded.into();
}
cards.push(data).unwrap();
}
rv["card_list"] = cards;
for deck in rv["deck_list"].members_mut() {
let mut used = array![];
for slot in deck["main_card_ids"].members_mut() {
let id = guest::proxy_card_id(slot.as_i64().unwrap_or(0));
// Cards the downgrade merged away leave the slot empty
if id == 0 || used.contains(id) {
*slot = (0).into();
continue;
}
used.push(id).unwrap();
*slot = id.into();
}
}
for key in ["favorite_master_card_id", "guest_smile_master_card_id", "guest_cool_master_card_id", "guest_pure_master_card_id"] {
let id = rv["user"][key].as_i64().unwrap_or(0);
rv["user"][key] = guest::proxy_card_id(id).into();
}
rv
}
*/
fn acc_exists(uid: i64) -> bool {
DATABASE.lock_and_select("SELECT user_id FROM userdata WHERE user_id=?1", params!(uid)).is_ok()
}
@@ -247,9 +199,6 @@ fn get_uid(token: &str) -> i64 {
data.parse::<i64>().unwrap_or(0)
}
// The account a login token belongs to, 0 when the token is unknown. The HTTP layer
// never needs this (it keys everything off the token itself), but the multi-live relay
// authenticates a {userId, token} pair and has to check the two agree.
pub fn uid_from_login_token(token: &str) -> i64 {
get_uid(token)
}
@@ -301,12 +250,7 @@ fn get_data(auth_key: &str, row: &str) -> JsonValue {
jzon::parse(&result.unwrap()).unwrap()
}
// Deleted custom songs leave stale score/clear records behind. They're wiped
// lazily when the userdata is pulled: collect the user's own music-id-keyed
// rows in the custom range (official ids are never candidates) and drop the
// ones whose id no longer exists in the catalog. Custom ids are never reused,
// so the wipe is final. A song that still exists but isn't visible to this
// user is NOT wiped - existence is what's checked, not visibility
// Prune deleted custom songs
fn remove_deleted_custom_songs(user: &mut JsonValue) -> bool {
// Feature off: never touch custom_songs.db, leave userdata untouched
if crate::router::custom_song::disabled() {
@@ -324,7 +268,9 @@ fn remove_deleted_custom_songs(user: &mut JsonValue) -> bool {
if candidates.is_empty() {
return false;
}
let dead = custom_song::dead_music_ids(&candidates);
let Some(dead) = custom_song::dead_music_ids(&candidates) else {
return false;
};
if dead.is_empty() {
return false;
}
@@ -341,13 +287,7 @@ fn remove_deleted_custom_songs(user: &mut JsonValue) -> bool {
true
}
// Deleted custom cards leave stale card_list rows behind - and a card_list id
// the client can't resolve aborts its whole login. Wiped lazily when the
// userdata is pulled, mirroring remove_deleted_custom_songs: only the runtime
// band is a candidate (official/imported ids never are), ids are never
// reused, so the wipe is final. A card that still exists but is unpublished
// is NOT wiped - existence is what's checked, and the catalog keeps serving
// owned ids (custom_card::owned_runtime_ids) so holders still resolve them
// Prune deleted custom cards
fn remove_deleted_custom_cards(user: &mut JsonValue) -> bool {
// Feature off: never touch custom_cards.db, leave userdata untouched
if crate::router::custom_card::disabled() {
@@ -363,7 +303,9 @@ fn remove_deleted_custom_cards(user: &mut JsonValue) -> bool {
if candidates.is_empty() {
return false;
}
let dead = custom_card::dead_card_ids(&candidates);
let Some(dead) = custom_card::dead_card_ids(&candidates) else {
return false;
};
if dead.is_empty() {
return false;
}
@@ -382,9 +324,6 @@ fn remove_deleted_custom_cards(user: &mut JsonValue) -> bool {
}
}
}
// A dead favorite/guest card repoints to the account's first remaining
// card (every account has its tutorial cards; the fallback can't trigger
// in practice)
let fallback = user["card_list"][0]["master_card_id"].as_i64().unwrap_or(10010001);
for key in ["favorite_master_card_id", "guest_smile_master_card_id", "guest_cool_master_card_id", "guest_pure_master_card_id"] {
if dead.contains(user["user"][key].as_i64().unwrap_or(0)) {
@@ -484,22 +423,6 @@ pub fn save_server_data(auth_key: &str, data: JsonValue) {
save_data(auth_key, "server_data", data);
}
// Read-modify-write of one account's server_data as ONE atomic step.
//
// get_server_data + save_server_data open a connection each, so two requests for the same
// account both read the pre-state and the later write wins - which for a record that is
// meant to be spent exactly once (the started-live record /multi_live/end awards off) means
// both ends see it and both award. Everything here happens inside a single BEGIN IMMEDIATE
// transaction instead, so concurrent callers serialise and the second one observes what the
// first one wrote.
//
// `f` sees the parsed server_data and returns whatever the caller needs out of it; the
// (possibly mutated) value is written back before the transaction commits. get_key runs
// BEFORE the transaction because it can create the account, which writes on its own
// connection and would otherwise deadlock against our write lock.
//
// A database error yields T::default() rather than a panic: the callers all have a
// "nothing to spend" branch, which is the right answer when the record could not be read.
pub fn modify_server_data<T: Default>(auth_key: &str, f: impl FnOnce(&mut JsonValue) -> T) -> T {
let key = get_key(auth_key);
let rv = DATABASE.lock_and_transact(|conn| {
@@ -788,15 +711,6 @@ pub fn export_user(token: &str) -> Option<JsonValue> {
})
}
// Every row an account owns, gone. Factored out of purge_accounts so the arcade
// sweeper (a machine that aged out takes its two accounts with it) and the
// card-rebind orphan cleanup delete exactly the same set of rows - an account
// half-deleted here is one the login path would resurrect empty.
//
// This list is also what the arcade guest reset mirrors: starter::write_starter_rows
// rewrites the eleven data rows, re-draws the token and deletes the rest, so a
// row added here has to be accounted for there too or a guest starts carrying
// the previous player's state across a credit.
pub const ACCOUNT_TABLES: &[&str] = &[
"userdata", "userhome", "missions", "loginbonus", "sifcards", "friends",
"chats", "exchange", "event", "eventloginbonus", "server_data", "webui",
@@ -808,11 +722,12 @@ pub fn delete_account(user_id: i64) {
for table in ACCOUNT_TABLES {
DATABASE.lock_and_exec(&format!("DELETE FROM {} WHERE user_id=?1", table), params!(user_id));
}
crate::router::custom_song::purge_owner(user_id);
crate::router::custom_card::purge_owner(user_id);
crate::router::custom_3dmv::purge_owner(user_id);
}
// True when the account has ever registered a data-transfer password, which is
// the only way an account can be taken over from another device. The arcade uses
// it to tell a throwaway account it made itself from a real player's account.
pub fn has_transfer_password(user_id: i64) -> bool {
!DATABASE.lock_and_select("SELECT password FROM migration WHERE user_id=?1", params!(user_id))
.unwrap_or_default()
@@ -841,6 +756,9 @@ pub fn purge_accounts() -> usize {
dead_uids.len()
}
// more tests???
#[cfg(test)]
mod tests {
use super::*;
@@ -855,10 +773,10 @@ mod tests {
let mut user = get_acc(token);
let deleted_id = custom_song::next_music_id();
custom_song::insert_song(deleted_id, 1, &object!{music_id: deleted_id}, "public", &array![], false);
custom_song::insert_song(deleted_id, 1, &object!{music_id: deleted_id}, "public", &array![], false).unwrap();
// Exists but isn't visible to this user - must survive the wipe
let private_id = custom_song::next_music_id();
custom_song::insert_song(private_id, 1, &object!{music_id: private_id}, "private", &array![], false);
custom_song::insert_song(private_id, 1, &object!{music_id: private_id}, "private", &array![], false).unwrap();
// 1100101 is a real stock live-id shape (7-digit, >= FIRST_MUSIC_ID): it
// must survive the custom-song wipe, unlike an unrealistic sub-10000 id
@@ -883,7 +801,7 @@ mod tests {
assert_eq!(user["live_list"].len(), 3);
assert_eq!(user["live_mission_list"].len(), 3);
custom_song::delete_song(deleted_id);
custom_song::delete_song(deleted_id).unwrap();
// The next pull drops the dead id's records and only those
let user = get_acc(token);
@@ -912,10 +830,10 @@ mod tests {
let mut user = get_acc(token);
let deleted_id = custom_card::next_card_id();
custom_card::insert_card(deleted_id, 1001, 1, &jzon::object!{ "master_card_id": deleted_id, "rarity": 1 }, true, true);
custom_card::insert_card(deleted_id, 1001, 1, &jzon::object!{ "master_card_id": deleted_id, "rarity": 1 }, true, true).unwrap();
// Exists but was unpublished - must survive the wipe
let unpublished_id = custom_card::next_card_id();
custom_card::insert_card(unpublished_id, 1001, 1, &jzon::object!{ "master_card_id": unpublished_id, "rarity": 1 }, false, false);
custom_card::insert_card(unpublished_id, 1001, 1, &jzon::object!{ "master_card_id": unpublished_id, "rarity": 1 }, false, false).unwrap();
for id in [deleted_id, unpublished_id] {
user["card_list"].push(jzon::object!{
+28 -10
View File
@@ -25,10 +25,13 @@ fn get_config() -> JsonValue {
pub fn get_login_token(req: &HttpRequest) -> Option<String> {
let blank_header = HeaderValue::from_static("");
let cookies = req.headers().get("Cookie").unwrap_or(&blank_header).to_str().unwrap_or("");
if cookies.is_empty() {
return None;
for pair in cookies.split(';') {
let Some((name, value)) = pair.split_once('=') else { continue; };
if name.trim() == "ew_token" {
return Some(value.trim().to_string());
}
Some(cookies.split("ew_token=").last().unwrap_or("").split(';').collect::<Vec<_>>()[0].to_string())
}
None
}
fn session_uid(req: &HttpRequest) -> Option<i64> {
@@ -607,13 +610,6 @@ pub fn remove_arcade_machine(req: HttpRequest, body: String) -> HttpResponse {
.body(jzon::stringify(resp))
}
// The account page's "bind arcade card" form: the card id, and nothing else.
// The webui session already proves whose account this is, so the card is bound
// to the signed-in account through arcade::bind_card_to - the same rule the
// cabinet's /api/arcade/bind applies once its own proof, the transfer code and
// password, has named the account. The cabinet endpoint speaks the encrypted
// game protocol behind the asset gate, which a browser cannot, so this is the
// browser's door onto that rule rather than a copy of it.
pub fn bind_arcade_card(req: HttpRequest, body: String) -> HttpResponse {
if crate::router::arcade::disabled() {
return HttpResponse::NotFound().finish();
@@ -687,6 +683,28 @@ pub fn cheat(req: HttpRequest, _body: String) -> HttpResponse {
mod tests {
use super::*;
// get_login_token is the ONLY authentication on every mutating custom-content
// route, so it has to read the cookie header as cookies, not as a substring:
// a name that merely ends in ew_token, or a second ew_token= appended later,
// must not win over the real session cookie
#[test]
fn the_session_cookie_is_matched_by_name() {
let token_for = |header: &str| get_login_token(
&actix_web::test::TestRequest::default().insert_header(("Cookie", header)).to_http_request()
);
assert_eq!(token_for("ew_token=real").as_deref(), Some("real"));
assert_eq!(token_for("theme=dark; ew_token=real; lang=en").as_deref(), Some("real"));
// A cookie whose NAME ends in ew_token is a different cookie
assert_eq!(token_for("not_ew_token=attacker").as_deref(), None);
assert_eq!(token_for("ew_token=real; xew_token=attacker").as_deref(), Some("real"));
// A duplicate set later in the header does not override the first
assert_eq!(token_for("ew_token=real; ew_token=attacker").as_deref(), Some("real"));
// No cookie at all is no session, not the whole header
assert_eq!(token_for("theme=dark").as_deref(), None);
assert_eq!(get_login_token(&actix_web::test::TestRequest::default().to_http_request()), None);
}
// The picker lists the card form searches by name: every baked character
// (official + SIF1 import, badged apart) and every skill_center row with
// its JP and EN display strings
+13 -20
View File
@@ -4,14 +4,20 @@ use jzon::{JsonValue, array};
pub struct SQLite {
path: String
}
const BUSY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
fn open(path: &str) -> Result<Connection, rusqlite::Error> {
let conn = Connection::open(path)?;
conn.busy_timeout(BUSY_TIMEOUT)?;
Ok(conn)
}
impl SQLite {
pub fn new(path: &str, setup: fn(&Connection)) -> SQLite {
let instance = SQLite {
path: crate::get_data_path(path)
};
let conn = Connection::open(&instance.path).unwrap();
conn.busy_timeout(std::time::Duration::from_secs(10)).unwrap();
let conn = open(&instance.path).unwrap();
conn.execute("PRAGMA foreign_keys = ON;", ()).unwrap();
setup(&conn);
instance
@@ -20,11 +26,11 @@ impl SQLite {
&self.path
}
pub fn lock_and_exec(&self, command: &str, args: &[&dyn ToSql]) {
let conn = Connection::open(&self.path).unwrap();
let conn = open(&self.path).unwrap();
conn.execute(command, args).unwrap();
}
pub fn lock_and_select(&self, command: &str, args: &[&dyn ToSql]) -> Result<String, rusqlite::Error> {
let conn = Connection::open(&self.path).unwrap();
let conn = open(&self.path)?;
let mut stmt = conn.prepare(command)?;
stmt.query_row(args, |row| {
match row.get::<usize, i64>(0) {
@@ -34,14 +40,14 @@ impl SQLite {
})
}
pub fn lock_and_select_type<T: rusqlite::types::FromSql>(&self, command: &str, args: &[&dyn ToSql]) -> Result<T, rusqlite::Error> {
let conn = Connection::open(&self.path).unwrap();
let conn = open(&self.path)?;
let mut stmt = conn.prepare(command)?;
stmt.query_row(args, |row| {
row.get(0)
})
}
pub fn lock_and_select_all(&self, command: &str, args: &[&dyn ToSql]) -> Result<JsonValue, rusqlite::Error> {
let conn = Connection::open(&self.path).unwrap();
let conn = open(&self.path)?;
let mut stmt = conn.prepare(command)?;
let map = stmt.query_map(args, |row| {
match row.get::<usize, i64>(0) {
@@ -60,24 +66,11 @@ impl SQLite {
Ok(rv)
}
// Runs a read-modify-write as one unit. Additive on purpose — the other helpers open
// a fresh connection per statement, so a caller that SELECTs then INSERTs races any
// concurrent caller doing the same and the loser hits a constraint violation (which
// lock_and_exec would unwrap into a worker panic).
//
// BEGIN IMMEDIATE takes the write lock up front rather than at first write, so two
// callers serialise instead of both reading the pre-state; busy_timeout makes the
// loser wait for the winner rather than fail instantly (SQLite::new sets that on its
// own short-lived setup connection, not on the per-call ones).
//
// Errors are returned, never unwrapped: statistics writes must not take down a
// request.
pub fn lock_and_transact<T>(
&self,
f: impl FnOnce(&Connection) -> Result<T, rusqlite::Error>
) -> Result<T, rusqlite::Error> {
let mut conn = Connection::open(&self.path)?;
conn.busy_timeout(std::time::Duration::from_secs(10))?;
let mut conn = open(&self.path)?;
let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
let rv = f(&tx)?;
tx.commit()?;
+1 -1
View File
@@ -71,7 +71,7 @@ fn platform_guard(ctx: &guard::GuardContext) -> bool {
.split('/')
.nth(1)
.unwrap_or("");
matches!(platform, "Android" | "StandaloneWindows64" | "StandaloneLinux64" | "WebGL" | "iOS")
matches!(platform, "Android" | "StandaloneWindows64" | "StandaloneLinux64" | "StandaloneOSX" | "WebGL" | "iOS")
}
pub fn routes(cfg: &mut web::ServiceConfig) {
+1 -1
Submodule webui updated: fa4696d2b5...bea8024e80