diff --git a/src/lib.rs b/src/lib.rs index 1ae827b..d82d120 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,6 +35,11 @@ pub async fn run_server(in_thread: bool) -> std::io::Result<()> { println!("Purged {} accounts", ct); } + // One-time, idempotent: charts stored before the custom-song spawn-group pairing rule are + // regrouped in place and their catalog md5s refreshed, so clients re-download the fixed + // encoding. No-op when custom songs are disabled or nothing needs rewriting. + router::custom_song::migrate::run(); + let rv = HttpServer::new(|| App::new() //.wrap(Cors::permissive()) .wrap_fn(|req, srv| { diff --git a/src/router/custom_card.rs b/src/router/custom_card.rs index d48ace7..92d344f 100644 --- a/src/router/custom_card.rs +++ b/src/router/custom_card.rs @@ -87,9 +87,12 @@ const CARD_TYPE_MAX: i64 = 4; const CARD_RARITY_MIN: i64 = 1; const CARD_RARITY_MAX: i64 = 3; const RARITY_NAMES: &[&str] = &["R", "SR", "UR"]; -// Sanity ceilings for the level-indexed skill arrays -const SKILL_PROBABILITY_MAX: i64 = 1_000_000; -const SKILL_MILLI_SECS_MAX: i64 = 600_000; +// Indexed by effect_type (0 unused) +const EFFECT_NAMES: &[&str] = &[ + "none", "smile p up", "pure p up", "cool p up", "score up", + "perfect window", "stamina recovery", "skill chance up", + "skill level boost", "param sync", "combo fever", "skill repeat" +]; struct ArtKind { kind: &'static str, @@ -144,6 +147,36 @@ const MAX_VOICE_SECONDS: f64 = 30.0; type Fields = HashMap>; +// new_skill's level-array columns come out of the csv layer as either +// [number] (a single plain value) or ["a/b/c"] (the slash-packed string) - +// flatten both into the numeric values +fn shipped_values(cell: &JsonValue) -> Vec { + let mut rv = Vec::new(); + let mut sources: Vec = Vec::new(); + for member in cell.members() { + if let Some(value) = member.as_i64() { + rv.push(value); + } else if let Some(s) = member.as_str() { + sources.push(s.to_string()); + } + } + if !cell.is_array() { + if let Some(value) = cell.as_i64() { + rv.push(value); + } else if let Some(s) = cell.as_str() { + sources.push(s.to_string()); + } + } + for source in sources { + for part in source.split('/') { + if let Ok(value) = part.trim().parse::() { + rv.push(value); + } + } + } + rv +} + lazy_static! { // Id allocation and the insert must not race between two uploads static ref UPLOAD_LOCK: Mutex<()> = Mutex::new(()); @@ -180,6 +213,70 @@ lazy_static! { rv }; + // Skill MAGNITUDE envelopes derived from the shipped new_skill.csv, the + // same way STAT_CAPS derives from card.csv. Enums are bounded elsewhere; + // these bound the numbers (a 40-second buff and a 1e8 score-up both got + // uploaded before this existed). Official rows (id < 100M) define the + // range; effect types no official row uses (1-3, 7-11 - the SIF1-port + // additions) fall back to the imported band's engineered range, flagged + // so the error message says which. (min, max, "official"/"imported") + static ref SKILL_VALUE_RANGES: HashMap = { + let mut official: HashMap = HashMap::new(); + let mut imported: HashMap = HashMap::new(); + for row in table(Region::Jp, "new_skill").members() { + let Some(id) = row["id"].as_i64() else { continue; }; + let Some(effect) = row["effectType"].as_i64() else { continue; }; + let into = if id < 100_000_000 { &mut official } else { &mut imported }; + for value in shipped_values(&row["effectiveValues"]) { + let entry = into.entry(effect).or_insert((value, value)); + entry.0 = entry.0.min(value); + entry.1 = entry.1.max(value); + } + } + let mut rv = HashMap::new(); + for (effect, (min, max)) in imported { + rv.insert(effect, (min, max, "imported")); + } + for (effect, (min, max)) in official { + rv.insert(effect, (min, max, "official")); + } + rv + }; + + // trigger -> (min, max) trigger_value over the official rows (trigger 4 = + // SECOND counts in milliseconds, hence its larger numbers) + static ref SKILL_TRIGGER_RANGES: HashMap = { + let mut rv: HashMap = HashMap::new(); + for row in table(Region::Jp, "new_skill").members() { + if row["id"].as_i64().unwrap_or(i64::MAX) >= 100_000_000 { continue; } + let Some(trigger) = row["trigger"].as_i64() else { continue; }; + for value in shipped_values(&row["triggerValue"]) { + let entry = rv.entry(trigger).or_insert((value, value)); + entry.0 = entry.0.min(value); + entry.1 = entry.1.max(value); + } + } + rv + }; + + // (probability, effective_milli_secs) global (min, max) over official rows + static ref SKILL_SCALAR_RANGES: ((i64, i64), (i64, i64)) = { + let mut prob = (i64::MAX, 0); + let mut ms = (i64::MAX, 0); + for row in table(Region::Jp, "new_skill").members() { + if row["id"].as_i64().unwrap_or(i64::MAX) >= 100_000_000 { continue; } + for value in shipped_values(&row["probability"]) { + prob.0 = prob.0.min(value); + prob.1 = prob.1.max(value); + } + for value in shipped_values(&row["effectiveMilliSecs"]) { + ms.0 = ms.0.min(value); + ms.1 = ms.1.max(value); + } + } + (prob, ms) + }; + // rarity -> (hp, smile, cool, pure) ceilings: the maximum any official // card of that rarity reaches. do_reinforce trusts the stored card // completely, so an uploaded stat is permanent - cap it at upload @@ -367,6 +464,24 @@ pub fn upload_limits() -> JsonValue { "name_en": row["nameEn"].clone() }).unwrap(); } + // The skill magnitude envelopes, so the form's number inputs carry real + // min/max and reject out-of-range (and e-notation) before submitting + let mut effect_value_ranges = object!{}; + for (effect, (min, max, source)) in SKILL_VALUE_RANGES.iter() { + effect_value_ranges[effect.to_string()] = object!{ + "min": *min, + "max": *max, + "source": *source + }; + } + let mut trigger_value_ranges = object!{}; + for (trigger, (min, max)) in SKILL_TRIGGER_RANGES.iter() { + trigger_value_ranges[trigger.to_string()] = object!{ + "min": *min, + "max": *max + }; + } + let ((prob_min, prob_max), (ms_min, ms_max)) = *SKILL_SCALAR_RANGES; object!{ "stat_caps": stat_caps, "skill_levels": skill_levels, @@ -377,8 +492,12 @@ pub fn upload_limits() -> JsonValue { "effect_type_max": SKILL_EFFECT_TYPE_MAX, "sub_target_max": SKILL_SUB_TARGET_MAX, "school_grade_max": SKILL_SCHOOL_GRADE_MAX, - "probability_max": SKILL_PROBABILITY_MAX, - "milli_secs_max": SKILL_MILLI_SECS_MAX, + "skill_ranges": { + "effect_values": effect_value_ranges, + "trigger_values": trigger_value_ranges, + "probability": { "min": prob_min, "max": prob_max }, + "milli_secs": { "min": ms_min, "max": ms_max } + }, "min_source_dim": art::MIN_SOURCE_DIM } } @@ -903,22 +1022,32 @@ fn build_card(master_card_id: i64, master_character_id: i64, fields: &Fields, st } // The client walks these in lockstep with the skill level, so each must - // carry exactly one value per level of the rarity's curve. Shipped + // carry exactly one value per level of the rarity's curve (shipped // masterdata also allows a single constant duration, so - // effective_milli_secs may be 1 long + // effective_milli_secs may be 1 long). MAGNITUDES are clamped to the + // ranges the shipped skill rows actually use - a 40-second buff and a + // 1e8 score-up both made it through before these bounds existed + let (trigger_min, trigger_max) = *SKILL_TRIGGER_RANGES.get(&trigger).unwrap_or(&(1, 1)); + let (value_min, value_max, value_source) = *SKILL_VALUE_RANGES.get(&effect_type).unwrap_or(&(1, 1, "official")); + let ((prob_min, prob_max), (ms_min, ms_max)) = *SKILL_SCALAR_RANGES; + let bounds = [ + ("skill_trigger_value", "trigger_value", trigger_min, trigger_max, + format!("for trigger {} (the official range)", trigger)), + ("skill_probability", "probability", prob_min, prob_max, + format!("({}%-{}%, the official range)", prob_min / 1000, prob_max / 1000)), + ("skill_effective_milli_secs", "effective_milli_secs", ms_min, ms_max, + String::from("milliseconds (the official range)")), + ("skill_effective_values", "effective_values", value_min, value_max, + format!("for effect_type {} ({}) (the {} range)", effect_type, EFFECT_NAMES[effect_type as usize], value_source)) + ]; let mut arrays: HashMap<&str, Vec> = HashMap::new(); - for (form, key, max) in [ - ("skill_trigger_value", "trigger_value", u32::MAX as i64), - ("skill_probability", "probability", SKILL_PROBABILITY_MAX), - ("skill_effective_milli_secs", "effective_milli_secs", SKILL_MILLI_SECS_MAX), - ("skill_effective_values", "effective_values", u32::MAX as i64) - ] { + for (form, key, min, max, label) in bounds { let values = levels_of(fields, form, &stored_skill, key)?; if values.len() != levels && !(key == "effective_milli_secs" && values.len() == 1) { return Err(format!("{} needs exactly {} values for a rarity {} card, got {}", form, levels, rarity, values.len())); } - if let Some(value) = values.iter().find(|v| **v > max) { - return Err(format!("{}: '{}' exceeds the maximum of {}", form, value, max)); + if let Some(value) = values.iter().find(|v| **v < min || **v > max) { + return Err(format!("{}: '{}' is out of range - values must be {}-{} {}", form, value, min, max, label)); } arrays.insert(key, values); } @@ -1624,6 +1753,31 @@ pub mod tests { wipe(4010); } + // Diagnostic + sanity guard for the derived skill magnitude envelopes + #[test] + fn skill_ranges_derive_from_shipped_rows() { + println!("effect_values:"); + for effect in 1..=11i64 { + let (min, max, source) = SKILL_VALUE_RANGES.get(&effect).copied().unwrap_or((0, 0, "MISSING")); + println!(" {} ({}): {}..{} [{}]", effect, EFFECT_NAMES[effect as usize], min, max, source); + assert!(min >= 1 && max >= min, "effect {} has no sane range", effect); + } + println!("trigger_values:"); + for trigger in 1..=4i64 { + let (min, max) = SKILL_TRIGGER_RANGES.get(&trigger).copied().unwrap_or((0, 0)); + println!(" {}: {}..{}", trigger, min, max); + assert!(min >= 1 && max >= min, "trigger {} has no sane range", trigger); + } + let ((prob_min, prob_max), (ms_min, ms_max)) = *SKILL_SCALAR_RANGES; + println!("probability: {}..{}", prob_min, prob_max); + println!("milli_secs: {}..{}", ms_min, ms_max); + // The two incidents must be outside whatever derives + assert!(prob_min >= 1000 && prob_max <= 100_000); + assert!(ms_min >= 500 && ms_max < 40_000, "40s buffs must be out of range"); + let (_, value_max, _) = SKILL_VALUE_RANGES[&4]; + assert!(value_max < 100_000_000, "1e8 score-ups must be out of range"); + } + // A full create: derived ids, pinned columns, art md5s and the catalog // shape the client parses #[test] @@ -1806,10 +1960,35 @@ pub mod tests { assert!(run(&mutated("skill_effective_values", "1/2/x")).unwrap_err().contains("not a number")); assert!(run(&mutated("skill_trigger_value", "")).unwrap_err().contains("skill_trigger_value")); assert!(run(&mutated("skill_effective_milli_secs", "2000")).is_ok()); - assert!(run(&mutated("skill_probability", "2000000/1/1")).unwrap_err().contains("maximum")); // Rarity 2 wants 5 entries, so the rarity-1 arrays no longer fit assert!(run(&mutated("rarity", "2")).unwrap_err().contains("needs exactly 5")); + // Skill MAGNITUDES clamp to the shipped ranges - both real incidents: + // the 40-second buff and the 1e8 score-up + let ms_err = run(&mutated("skill_effective_milli_secs", "40000/40000/40000")).unwrap_err(); + assert!(ms_err.contains("skill_effective_milli_secs") && ms_err.contains("2000-6000"), "{}", ms_err); + let value_err = run(&mutated("skill_effective_values", "100000000/100000000/100000000")).unwrap_err(); + assert!(value_err.contains("91-439") && value_err.contains("score up") && value_err.contains("official"), "{}", value_err); + // Literal e-notation never even parses as an integer + assert!(run(&mutated("skill_effective_values", "1e8/1e8/1e8")).unwrap_err().contains("not a number")); + // Boundary accept/reject for effect 4 (score up), official 91-439 + assert!(run(&mutated("skill_effective_values", "91/200/439")).is_ok()); + assert!(run(&mutated("skill_effective_values", "91/200/440")).unwrap_err().contains("out of range")); + assert!(run(&mutated("skill_effective_values", "90/200/439")).unwrap_err().contains("out of range")); + // A SIF1-port effect type bounds to the imported band's range + let mut fields = base_fields(); + field(&mut fields, "skill_effect_type", "5"); + field(&mut fields, "skill_effective_values", "1/2/2"); + assert!(run(&fields).is_ok()); + field(&mut fields, "skill_effect_type", "7"); + let err = run(&fields).unwrap_err(); + assert!(err.contains("1090-2600") && err.contains("imported"), "{}", err); + // Trigger values bound per trigger (trigger 3 = PERFECTs, 13-30) and + // probability to the official 16%-75% + assert!(run(&mutated("skill_trigger_value", "13/20/31")).unwrap_err().contains("13-30")); + assert!(run(&mutated("skill_probability", "80000/39000/39000")).unwrap_err().contains("16000-75000")); + assert!(run(&mutated("skill_probability", "15999/39000/39000")).unwrap_err().contains("out of range")); + // Stat bounds, computed from the official card.csv: hp really is a // tiny per-rarity constant (R 2 / SR 3 / UR 4), so an R card allows // 1-2 - and the message says so instead of hiding the limit @@ -1871,8 +2050,8 @@ pub mod tests { fields.insert(String::from("pr"), seeded_png(32, 32, 9)); assert!(runc(&fields).unwrap_err().contains("at least")); - // Only the two deliberate successes above wrote rows - assert_eq!(database::card_count_for_owner(4004), 2); + // Only the deliberate successes above wrote rows + assert_eq!(database::card_count_for_owner(4004), 4); wipe(4004); } @@ -1930,10 +2109,12 @@ pub mod tests { let before = database::get_card(id).unwrap(); // The sc_00 override is deliberately the wrong size: an update crops - // it to target just like create does + // it to target just like create does. The effect change brings values + // in that effect's shipped range along (the old ones are out of it) let mut edit = Fields::new(); field(&mut edit, "name", "Renamed"); field(&mut edit, "skill_effect_type", "7"); + field(&mut edit, "skill_effective_values", "1090/2000/2600"); edit.insert(String::from("sc_00"), seeded_png(700, 900, 99)); with_permissions(4007, &[permissions::CARD_UPLOAD], || update_card(4007, id, &edit).unwrap()); diff --git a/src/router/custom_song.rs b/src/router/custom_song.rs index cd2b909..cb4c0b2 100644 --- a/src/router/custom_song.rs +++ b/src/router/custom_song.rs @@ -2,6 +2,9 @@ // in-process symphonia + vorbis machinery pub mod audio; mod chart; +// One-time startup regroup of charts stored before the spawn-group pairing rule; called from +// run_server, no-op when the feature is disabled or every chart is already correctly grouped +pub mod migrate; mod package; use jzon::{array, object, JsonValue}; @@ -1015,6 +1018,102 @@ mod tests { fields.insert(String::from(key), value.as_bytes().to_vec()); } + // A SIF1 chart whose transcode contains 3+ simultaneous notes: 4 parallel holds into a + // full 9-lane wall (the shape of the field-reported chart that exposed the old encoding) + fn wall_chart() -> Vec { + let mut beatmap = jzon::array![]; + for position in [2, 4, 6, 8] { + beatmap.push(jzon::object!{ + "timing_sec": 1.0, "notes_attribute": 1, "notes_level": 1, + "effect": 3, "effect_value": 1.0, "position": position + }).unwrap(); + } + for position in 1..=9 { + beatmap.push(jzon::object!{ + "timing_sec": 2.75, "notes_attribute": 1, "notes_level": 1, + "effect": 1, "effect_value": 0.0, "position": position + }).unwrap(); + } + jzon::stringify(beatmap).into_bytes() + } + + // The startup migration: a chart stored with the PRE-pairing encoding (whole equal-time + // clusters sharing one num) is regrouped in place, its catalog md5/size follow the new + // bytes, the revision bumps exactly once, correctly-encoded songs stay byte-identical, + // and a second run is a complete no-op + #[test] + fn startup_migration_regroups_pre_fix_charts() { + let _lock = crate::runtime::lock_test_data_path(); + + let mut fields = HashMap::new(); + field(&mut fields, "name", "Migration Target"); + field(&mut fields, "artist", "Wall Artist"); + field(&mut fields, "attribute", "1"); + field(&mut fields, "level_number_4", "15"); + fields.insert(String::from("jacket"), test_png()); + fields.insert(String::from("audio"), test_ogg_tone(550.0)); + fields.insert(String::from("chart_4"), wall_chart()); + let target = create_song(3333, &fields).unwrap(); + + let mut fields = HashMap::new(); + field(&mut fields, "name", "Migration Control"); + field(&mut fields, "artist", "Control Artist"); + field(&mut fields, "attribute", "2"); + field(&mut fields, "level_number_1", "5"); + fields.insert(String::from("jacket"), test_png()); + fields.insert(String::from("audio"), test_ogg_tone(770.0)); + fields.insert(String::from("chart_1"), test_chart()); + let control = create_song(4444, &fields).unwrap(); + + // The upload stored the CURRENT encoding; capture it, then doctor the store back to + // the pre-pairing form exactly as an old server would have written it: squashed + // chart bytes on disk and the catalog md5/size matching those bytes + let path = song_path(target, "chart_4.json"); + let fixed_bytes = fs::read(&path).unwrap(); + let mut squashed = jzon::parse(&String::from_utf8_lossy(&fixed_bytes)).unwrap(); + chart::squash_to_pre_fix(&mut squashed); + let squashed_bytes = jzon::stringify(squashed).into_bytes(); + assert_ne!(squashed_bytes, fixed_bytes); + fs::write(&path, &squashed_bytes).unwrap(); + let mut song = database::get_song(target).unwrap(); + let (md5, size) = asset_meta(&squashed_bytes); + for entry in song["levels"].members_mut() { + if entry["level"] == 4 { + entry["md5"] = md5.clone().into(); + entry["size"] = size.into(); + } + } + database::update_song(target, &song); + + let control_bytes = fs::read(song_path(control, "chart_1.json")).unwrap(); + let control_song = database::get_song(control).unwrap(); + let revision = database::get_revision(); + + migrate::run(); + + // The target chart is byte-identical to what the current transcoder stores, and the + // catalog follows the new bytes + let migrated = fs::read(&path).unwrap(); + assert_eq!(migrated, fixed_bytes); + let song = database::get_song(target).unwrap(); + let level = song["levels"].members().find(|l| l["level"] == 4).unwrap(); + let (md5, size) = asset_meta(&fixed_bytes); + assert_eq!(level["md5"].to_string(), md5); + assert_eq!(level["size"].as_usize().unwrap(), size); + // full_combo never depended on grouping and must not move + assert_eq!(level["full_combo"], 9 + 4); + + // Exactly one revision bump, and the control song is untouched + assert_eq!(database::get_revision(), revision + 1); + assert_eq!(fs::read(song_path(control, "chart_1.json")).unwrap(), control_bytes); + assert_eq!(jzon::stringify(database::get_song(control).unwrap()), jzon::stringify(control_song)); + + // Idempotent: a second boot changes nothing and bumps nothing + migrate::run(); + assert_eq!(fs::read(&path).unwrap(), fixed_bytes); + assert_eq!(database::get_revision(), revision + 1); + } + // Export a song, import the package as another user, and the served song // must be identical apart from the assigned music_id - INCLUDING the audio // md5s: ogg uploads are stored as-is and the preview encode is diff --git a/src/router/custom_song/chart.rs b/src/router/custom_song/chart.rs index a15e26d..5e3fd9e 100644 --- a/src/router/custom_song/chart.rs +++ b/src/router/custom_song/chart.rs @@ -262,6 +262,113 @@ pub fn transcode(beatmap: &JsonValue) -> Result<(JsonValue, i64), String> { }, max_combo_count)) } +// Regroups a STORED transcoded chart whose spawn groups predate the pairing rule above: the +// old transcoder gave every note of an equal-time cluster one shared num (force_sync_group_id +// always 0), and the client renders no head markers for a group of 3+ (see the header +// comment). This rebuilds num / force_sync_group_id in place with the same clustering the +// transcoder now uses — equal final time, lane-sorted, chunks of two, chained +// force_sync_group_id — and re-points child_num at each child's renumbered spawn group. +// Everything else (ids, times, lines, types, parent/child links, max_combo_count — combo +// counting never depended on grouping) is untouched, so on a chart the current transcoder +// produced this reproduces the stored bytes exactly. +// +// Returns false (chart untouched) unless some num is shared by MORE than two notes. That +// makes it a safe no-op on current uploads AND on official-style encodings (whose num values +// differ from ours — e.g. gaps of 3 — but whose groups never exceed two). +pub fn regroup(chart: &mut JsonValue) -> bool { + // (id, time, line) per real note; the dummy header at [0] stays untouched + let notes: Vec<(i64, f64, i64)> = chart["notes"].members().skip(1).map(|n| ( + n["id"].as_i64().unwrap_or(0), + n["time"].as_f64().unwrap_or(0.0), + n["line"].as_i64().unwrap_or(0) + )).collect(); + + // Only a pre-pairing chart (some num shared 3+ ways) is rewritten + let mut group_sizes: Vec<(i64, i64)> = Vec::new(); + for data in chart["notes"].members().skip(1) { + let num = data["num"].as_i64().unwrap_or(0); + match group_sizes.iter_mut().find(|(n, _)| *n == num) { + Some((_, count)) => *count += 1, + None => group_sizes.push((num, 1)) + } + } + if group_sizes.iter().all(|(_, count)| *count <= 2) { + return false; + } + + // Time order; ids break ties (transcode issues them in time order, so this reproduces + // the emission order the grouping pass originally saw) + let mut order: Vec = (0..notes.len()).collect(); + order.sort_by(|a, b| notes[*a].1.total_cmp(¬es[*b].1).then(notes[*a].0.cmp(¬es[*b].0))); + + // id -> (new num, new force_sync_group_id) + let mut assigned: Vec<(i64, i64, i64)> = Vec::with_capacity(notes.len()); + let mut num = 100; + let mut start = 0; + while start < order.len() { + let mut end = start + 1; + while end < order.len() && notes[order[end]].1 == notes[order[start]].1 { + end += 1; + } + let mut cluster: Vec = order[start..end].to_vec(); + cluster.sort_by_key(|index| notes[*index].2); + let mut prev_num = 0; + for chunk in cluster.chunks(2) { + num += 1; + for index in chunk { + assigned.push((notes[*index].0, num, prev_num)); + } + prev_num = num; + } + start = end; + } + let lookup = |id: i64| assigned.iter().find(|(i, _, _)| *i == id).map(|(_, n, f)| (*n, *f)); + + for data in chart["notes"].members_mut().skip(1) { + let Some((new_num, force)) = lookup(data["id"].as_i64().unwrap_or(0)) else { continue; }; + data["num"] = new_num.into(); + data["force_sync_group_id"] = force.into(); + let child = data["child_id"].as_i64().unwrap_or(0); + if child != 0 { + // child_num names the child's spawn group and must follow its new num + data["child_num"] = lookup(child).map(|(n, _)| n).unwrap_or(0).into(); + } + } + true +} + +// Test helper: fabricates what pre-pairing servers stored, by squashing a current chart back +// to the OLD encoding — one shared num per equal-time cluster, force_sync_group_id 0, and +// child_num following. Lives outside the tests module so the migration tests in +// router/custom_song.rs can build realistic pre-fix fixtures from transcode output. +#[cfg(test)] +pub fn squash_to_pre_fix(chart: &mut JsonValue) { + let notes: Vec<(i64, f64)> = chart["notes"].members().skip(1) + .map(|n| (n["id"].as_i64().unwrap(), n["time"].as_f64().unwrap())) + .collect(); + let mut order: Vec = (0..notes.len()).collect(); + order.sort_by(|a, b| notes[*a].1.total_cmp(¬es[*b].1).then(notes[*a].0.cmp(¬es[*b].0))); + let mut nums: Vec<(i64, i64)> = Vec::new(); + let mut num = 100; + let mut last_time = f64::NEG_INFINITY; + for index in order { + if notes[index].1 != last_time { + num += 1; + last_time = notes[index].1; + } + nums.push((notes[index].0, num)); + } + let lookup = |id: i64| nums.iter().find(|(i, _)| *i == id).map(|(_, n)| *n).unwrap_or(0); + for data in chart["notes"].members_mut().skip(1) { + data["num"] = lookup(data["id"].as_i64().unwrap()).into(); + data["force_sync_group_id"] = 0.into(); + let child = data["child_id"].as_i64().unwrap_or(0); + if child != 0 { + data["child_num"] = lookup(child).into(); + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -657,6 +764,94 @@ mod tests { } } + // The 10011 field shape: 4 parallel holds into a full 9-lane wall into a triple. Squashing + // transcode output reproduces the old encoding exactly (one num per cluster, no force + // links); regroup must restore the current encoding BYTE-IDENTICALLY, child_num included. + #[test] + fn regroup_restores_pre_fix_wall_and_parallel_holds() { + let beatmap = jzon::array![ + sif_note(1.0, 2, 3, 1.0), sif_note(1.0, 4, 3, 1.0), + sif_note(1.0, 6, 3, 1.0), sif_note(1.0, 8, 3, 1.0), + sif_note(2.75, 1, 1, 0.0), sif_note(2.75, 2, 1, 0.0), sif_note(2.75, 3, 1, 0.0), + sif_note(2.75, 4, 1, 0.0), sif_note(2.75, 5, 1, 0.0), sif_note(2.75, 6, 1, 0.0), + sif_note(2.75, 7, 1, 0.0), sif_note(2.75, 8, 1, 0.0), sif_note(2.75, 9, 1, 0.0), + sif_note(3.625, 3, 1, 0.0), sif_note(3.625, 5, 1, 0.0), sif_note(3.625, 7, 1, 0.0) + ]; + let (expected, _) = transcode(&beatmap).unwrap(); + + let mut chart = expected.clone(); + squash_to_pre_fix(&mut chart); + // Sanity: the squash really is the old encoding — whole clusters share one num + assert_eq!(chart["notes"][1]["num"], 101); + assert_eq!(chart["notes"][4]["num"], 101); // all 4 hold heads + assert_eq!(chart["notes"][9]["num"], 103); + assert_eq!(chart["notes"][17]["num"], 103); // all 9 wall notes + assert_eq!(chart["notes"][1]["child_num"], 102, "squashed child_num must follow"); + for data in chart["notes"].members() { + assert_eq!(data["force_sync_group_id"], 0); + } + + assert!(regroup(&mut chart), "a squashed chart must be rewritten"); + assert_eq!(jzon::stringify(chart.clone()), jzon::stringify(expected.clone()), + "regroup must reproduce the current transcoder's output exactly"); + + // Spell the wall out: adjacent-lane pairs, each later chunk force-synced to the + // previous one (heads 101/102, tails 103/104, wall 105..109, triple 110/111) + assert_spawn_groups_hold_at_most_two(&chart); + let wall: Vec<(i64, i64, i64)> = chart["notes"].members() + .filter(|d| d["time"].as_f64() == Some(2.75)) + .map(|d| (d["line"].as_i64().unwrap(), d["num"].as_i64().unwrap(), d["force_sync_group_id"].as_i64().unwrap())) + .collect(); + let mut wall_sorted = wall.clone(); + wall_sorted.sort(); + assert_eq!(wall_sorted, vec![ + (0, 105, 0), (1, 105, 0), + (2, 106, 105), (3, 106, 105), + (4, 107, 106), (5, 107, 106), + (6, 108, 107), (7, 108, 107), + (8, 109, 108) + ]); + // child_num follows the child's NEW num: each head's child_num names a tail group + for head in chart["notes"].members().filter(|d| d["time"].as_f64() == Some(1.0)) { + let child_id = head["child_id"].as_i64().unwrap(); + let tail = chart["notes"].members().find(|d| d["id"].as_i64() == Some(child_id)).unwrap(); + assert_eq!(head["child_num"], tail["num"].clone()); + assert!([103, 104].contains(&tail["num"].as_i64().unwrap())); + } + } + + #[test] + fn regroup_is_a_no_op_on_current_encoding() { + let beatmap = jzon::array![ + sif_note(1.0, 2, 1, 0.0), sif_note(1.0, 5, 1, 0.0), sif_note(1.0, 8, 1, 0.0), + sif_note(2.0, 4, 3, 1.5) + ]; + let (chart, _) = transcode(&beatmap).unwrap(); + let before = jzon::stringify(chart.clone()); + let mut chart = chart; + assert!(!regroup(&mut chart)); + assert_eq!(jzon::stringify(chart), before); + } + + // Official-shaped encodings (1132_5_Sn t=9.781: num gaps of 3, force_sync naming the other + // pair) have groups of at most two and must never be "normalized" to our num sequence + #[test] + fn regroup_is_a_no_op_on_official_shaped_charts() { + let mut chart = object!{ + "max_lane": 9, "sound_name": "", "max_combo_count": 4, + "notes": [ + {"id": 0, "num": 100, "line": 0, "time": 0.0, "type": 0, "parent_id": 0, "child_id": 0, "child_num": 0, "child_line": 0, "force_sync_group_id": 0}, + {"id": 45, "num": 145, "line": 0, "time": 9.781, "type": 1, "parent_id": 0, "child_id": 0, "child_num": 0, "child_line": 0, "force_sync_group_id": 0}, + {"id": 46, "num": 145, "line": 1, "time": 9.781, "type": 1, "parent_id": 0, "child_id": 0, "child_num": 0, "child_line": 0, "force_sync_group_id": 0}, + {"id": 47, "num": 148, "line": 7, "time": 9.781, "type": 1, "parent_id": 0, "child_id": 0, "child_num": 0, "child_line": 0, "force_sync_group_id": 145}, + {"id": 48, "num": 148, "line": 8, "time": 9.781, "type": 1, "parent_id": 0, "child_id": 0, "child_num": 0, "child_line": 0, "force_sync_group_id": 145} + ] + }; + let before = jzon::stringify(chart.clone()); + assert!(!regroup(&mut chart)); + assert_eq!(jzon::stringify(chart), before); + } + #[test] fn rejects_bad_charts() { assert!(transcode(&jzon::array![sif_note(1.0, 0, 1, 2.0)]).is_err()); diff --git a/src/router/custom_song/migrate.rs b/src/router/custom_song/migrate.rs new file mode 100644 index 0000000..ca8e70b --- /dev/null +++ b/src/router/custom_song/migrate.rs @@ -0,0 +1,80 @@ +use std::fs; + +use super::{chart, database, song_path, asset_meta, LEVEL_COUNT}; +use crate::runtime::get_data_path; + +// One-time, idempotent startup migration for charts transcoded before the spawn-group pairing +// rule (chart.rs header): the old transcoder gave every note of an equal-time cluster ONE +// shared num, and the client creates no head markers for a group of 3+ notes +// (LiveMarkerControl.CreateMarkerUI plain-returns on count > 2), so 3+ simultaneous notes +// were judged but never rendered. Stored transcoded charts are derived data with everything +// the regroup needs (time + line per note), so they are rewritten in place — no original +// upload required, which also covers songs from before export support. +// +// For each song directory with a catalog row, every chart with an over-shared num is +// regrouped (chart::regroup), rewritten to disk, and its level's md5/size in the catalog +// blob updated — the changed md5 re-keys the client's content-addressed cache, so clients +// re-download the fixed chart on their next catalog sync. The revision is bumped ONCE if +// anything changed. Charts the current transcoder produced (and official-shaped encodings) +// are left byte-identical, so re-running every boot is free. +pub fn run() { + if super::disabled() { + return; + } + let Ok(entries) = fs::read_dir(get_data_path("custom_songs")) else { + // No custom_songs directory yet - nothing was ever uploaded + return; + }; + + let mut music_ids: Vec = entries.flatten() + .filter_map(|entry| entry.file_name().to_string_lossy().parse::().ok()) + .collect(); + music_ids.sort(); + + let mut songs_changed = 0; + let mut charts_changed = 0; + for music_id in music_ids { + // A song directory without a catalog row is never served; leave it alone + let Some(mut song) = database::get_song(music_id) else { continue; }; + + let mut changed = false; + for level in 1..=LEVEL_COUNT { + let path = song_path(music_id, &format!("chart_{}.json", level)); + let Ok(bytes) = fs::read(&path) else { continue; }; + let Ok(mut chart_data) = jzon::parse(&String::from_utf8_lossy(&bytes)) else { + println!("Custom song {} chart {}: not valid JSON, migration skipped", music_id, level); + continue; + }; + if !chart::regroup(&mut chart_data) { + continue; + } + let new_bytes = jzon::stringify(chart_data); + if let Err(e) = fs::write(&path, &new_bytes) { + println!("Custom song {} chart {}: rewrite failed ({}), migration skipped", music_id, level, e); + continue; + } + // The catalog md5/size must follow the served bytes or the client's + // download-and-verify loop would never accept the asset + let (md5, size) = asset_meta(new_bytes.as_bytes()); + for entry in song["levels"].members_mut() { + if entry["level"] == level { + entry["md5"] = md5.clone().into(); + entry["size"] = size.into(); + } + } + changed = true; + charts_changed += 1; + println!("Custom song {}: regrouped chart level {} (pre-pairing spawn groups)", music_id, level); + } + + if changed { + database::update_song(music_id, &song); + songs_changed += 1; + } + } + + if songs_changed > 0 { + database::bump_revision(); + println!("Custom song spawn-group migration: rewrote {} chart(s) in {} song(s), catalog revision bumped", charts_changed, songs_changed); + } +} diff --git a/webui b/webui index 28b1c65..124065a 160000 --- a/webui +++ b/webui @@ -1 +1 @@ -Subproject commit 28b1c65fc6de15b3bcd88c31cb46eae6d727f9a5 +Subproject commit 124065a3138dcb64188823b2b07cb101dfa69d62