Add some more checks

src/database/custom_song.rs

@@ -303,6 +303,14 @@ pub fn dead_music_ids(candidates: &JsonValue) -> JsonValue {
     rv
 }
 
+// Every stored catalog blob, unparsed and unfiltered by visibility. Only the
+// startup audio sweep needs the whole table, and it needs to tell "no songs"
+// apart from "could not read the songs" - a read failure must never look like
+// an empty catalog, so this returns None rather than an empty array on error
+pub fn all_song_blobs() -> Option<JsonValue> {
+    DATABASE.lock_and_select_all("SELECT song FROM songs ORDER BY music_id", params!()).ok()
+}
+
 // Audio files are content-addressed and may be shared between songs
 pub fn audio_in_use(md5: &str, ignored_music_id: i64) -> bool {
     DATABASE.lock_and_select("SELECT music_id FROM songs WHERE music_id!=?1 AND song LIKE ?2", params!(ignored_music_id, format!("%{}%", md5))).is_ok()

src/lib.rs

@@ -35,10 +35,8 @@ pub async fn run_server(in_thread: bool) -> std::io::Result<()> {
         println!("Purged {} accounts", ct);
     }
 
-    // One-time, idempotent: charts stored before the custom-song spawn-group pairing rule are
-    // regrouped in place and their catalog md5s refreshed, so clients re-download the fixed
-    // encoding. No-op when custom songs are disabled or nothing needs rewriting.
     router::custom_song::migrate::run();
+    router::custom_song::sweep_audio();
 
     let rv = HttpServer::new(|| App::new()
     //.wrap(Cors::permissive())

src/router/custom_song.rs

@@ -160,6 +160,68 @@ fn audio_file_path(md5: &str) -> String {
     get_data_path(&format!("custom_songs/audio/{}.ogg", md5))
 }
 
+// Startup GC for the content-addressed audio store. Every ogg in there belongs
+// to some song's play or select cue; the only things that write the directory
+// are upload and update, so an unreferenced file is a leftover from an
+// interrupted one. Deleting it loses nothing - the same audio uploaded again
+// re-encodes to the same bytes at the same path (audio.rs).
+//
+// Deliberately conservative, because the failure mode is deleting audio a live
+// song serves: anything that makes the reference set doubtful - an unreadable
+// catalog, a blob that won't parse, a song with no cue md5 - aborts the whole
+// sweep instead of treating that song as referencing nothing. Only exactly
+// {32 hex}.ogg names are ever considered. Takes UPLOAD_LOCK for the same
+// reason delete does: an upload that has written its oggs but not yet inserted
+// its row must not look like an orphan.
+pub fn sweep_audio() {
+    if disabled() {
+        return;
+    }
+    let lock = lock_onto_mutex!(UPLOAD_LOCK);
+    let Some(blobs) = database::all_song_blobs() else {
+        println!("Custom song audio sweep: catalog unreadable, skipped");
+        return;
+    };
+    let mut referenced: Vec<String> = Vec::new();
+    for blob in blobs.members() {
+        let Ok(song) = jzon::parse(&blob.to_string()) else {
+            println!("Custom song audio sweep: unparseable catalog row, skipped");
+            return;
+        };
+        for key in ["play", "select"] {
+            let md5 = song["sound"][key]["md5"].as_str().unwrap_or("");
+            if md5.is_empty() {
+                println!("Custom song audio sweep: song {} has no {} cue, skipped", song["music_id"], key);
+                return;
+            }
+            referenced.push(String::from(md5));
+        }
+    }
+
+    // No directory means nothing was ever uploaded
+    let Ok(entries) = fs::read_dir(get_data_path("custom_songs/audio")) else {
+        return;
+    };
+    let mut removed = 0;
+    for entry in entries.flatten() {
+        let name = entry.file_name().to_string_lossy().to_string();
+        let Some(md5) = name.strip_suffix(".ogg") else { continue; };
+        if md5.len() != 32 || !md5.chars().all(|c| c.is_ascii_hexdigit()) {
+            continue;
+        }
+        if referenced.iter().any(|other| other == md5) {
+            continue;
+        }
+        if fs::remove_file(entry.path()).is_ok() {
+            removed += 1;
+        }
+    }
+    if removed > 0 {
+        println!("Custom song audio sweep: removed {} orphaned ogg(s)", removed);
+    }
+    drop(lock);
+}
+
 async fn assets(req: HttpRequest) -> HttpResponse {
     if disabled() {
         return HttpResponse::NotFound().finish();
@@ -1063,6 +1125,11 @@ async fn delete(req: HttpRequest, body: String) -> HttpResponse {
     }
     let song = database::get_song(music_id).unwrap_or(object!{});
 
+    // Same lock upload and update take, for the audio GC below: an upload
+    // writes its oggs before it inserts its row, so a delete that reads the
+    // catalog in that window sees no reference to a shared md5 and would
+    // unlink the file the new song is about to serve
+    let lock = lock_onto_mutex!(UPLOAD_LOCK);
     database::delete_song(music_id);
     database::bump_revision();
     // Global clear-rate stats for the dead live id (per-user score records are
@@ -1077,6 +1144,7 @@ async fn delete(req: HttpRequest, body: String) -> HttpResponse {
             let _ = fs::remove_file(audio_file_path(&md5));
         }
     }
+    drop(lock);
 
     send_json(object!{
         result: "OK"
@@ -1643,6 +1711,60 @@ mod tests {
         assert!(fs::read(audio_file_path(&before["sound"]["select"]["md5"].to_string())).is_err());
     }
 
+    // The startup sweep drops oggs nothing references any more (an upload that
+    // died between writing the file and storing its row), keeps every cue that
+    // is referenced - including one shared by two songs - and leaves anything
+    // that isn't a content-addressed ogg alone
+    #[test]
+    fn audio_sweep_removes_only_orphans() {
+        let _lock = crate::runtime::lock_test_data_path();
+
+        let mut fields = HashMap::new();
+        field(&mut fields, "name", "Sweep Song");
+        field(&mut fields, "artist", "Sweep Artist");
+        field(&mut fields, "attribute", "1");
+        fields.insert(String::from("jacket"), test_png());
+        fields.insert(String::from("audio"), test_ogg_tone(311.0));
+        fields.insert(String::from("chart_1"), test_chart());
+        let music_id = create_song(4242, &fields).unwrap();
+        // A second song on the SAME audio: identical bytes hash to one shared file
+        let shared_id = create_song(4243, &fields).unwrap();
+        let song = database::get_song(music_id).unwrap();
+        let play = song["sound"]["play"]["md5"].to_string();
+        let select = song["sound"]["select"]["md5"].to_string();
+        assert_eq!(database::get_song(shared_id).unwrap()["sound"]["play"]["md5"], play);
+
+        let orphan = audio_file_path(&"a1".repeat(16));
+        let not_an_ogg = get_data_path("custom_songs/audio/notes.txt");
+        let bad_name = get_data_path("custom_songs/audio/orphan.ogg");
+        for path in [&orphan, &not_an_ogg, &bad_name] {
+            fs::write(path, b"junk").unwrap();
+        }
+
+        sweep_audio();
+
+        assert!(fs::read(&orphan).is_err());
+        assert!(fs::read(audio_file_path(&play)).is_ok());
+        assert!(fs::read(audio_file_path(&select)).is_ok());
+        // Only {32 hex}.ogg is ever a sweep candidate
+        assert!(fs::read(&not_an_ogg).is_ok());
+        assert!(fs::read(&bad_name).is_ok());
+
+        // A song whose cues can't be read makes the whole sweep bail rather
+        // than treat that song as referencing nothing
+        fs::write(&orphan, b"junk").unwrap();
+        let shared_song = database::get_song(shared_id).unwrap();
+        database::update_song(shared_id, &object!{"music_id": shared_id});
+        sweep_audio();
+        assert!(fs::read(&orphan).is_ok());
+        assert!(fs::read(audio_file_path(&play)).is_ok());
+
+        database::update_song(shared_id, &shared_song);
+        let _ = fs::remove_file(&orphan);
+        let _ = fs::remove_file(&not_an_ogg);
+        let _ = fs::remove_file(&bad_name);
+    }
+
     // mp3/wav uploads still work: symphonia decodes them and the play cue is
     // transcoded to ogg-vorbis in-process
     #[test]