Adjust mod multiplier recalculation processes to minimise number of required database writes

osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/PopulateTotalScoreWithoutModsCommand.cs

@@ -10,8 +10,9 @@
 using JetBrains.Annotations;
 using McMaster.Extensions.CommandLineUtils;
 using MySqlConnector;
-using osu.Game.Scoring.Legacy;
+using osu.Game.Rulesets.Scoring;
 using osu.Server.QueueProcessor;
+using osu.Server.Queues.ScoreStatisticsProcessor.Helpers;
 using osu.Server.Queues.ScoreStatisticsProcessor.Models;
 
 namespace osu.Server.Queues.ScoreStatisticsProcessor.Commands.Maintenance
@@ -51,19 +52,35 @@ public async Task<int> OnExecuteAsync(CancellationToken cancellationToken)
 
             while (!cancellationToken.IsCancellationRequested)
             {
-                var scoresWithMissing = (await conn.QueryAsync<SoloScore>(
-                    // WARNING: the query below MUST use `JSON_VALUE` because mysql is being weird with the standard arrow syntax.
+                var scoresToPopulate = (await conn.QueryAsync<SoloScore>(
+                    // conditions for backpopulation:
+                    // - score must not be legacy.
+                    //   the totals of legacy scores are already computed from `legacy_total_score` via `StandardisedScoreMigrationTools`,
+                    //   so populating `total_score_without_mods` is unnecessary in that case, `legacy_total_score` is sufficient for lossless mod multiplier changes.
+                    // - score must have mods.
+                    //   it is assumed that a score without mods cannot have any other score multiplier than 1.0x,
+                    //   and therefore it would hold that `total_score_without_mods` == `total_score`.
+                    // - score must not have total score without mods already populated.
+                    //
+                    // WARNING: the query below MUST use `JSON_VALUE` to read `total_score_without_mods` because mysql is being weird with the standard arrow syntax.
                     // if `data` does not have `total_score_without_mods` set at all, `data->'$.total_score_without_mods'` will return the typical SQL NULL,
                     // but if `data` has `{'total_score_without_mods': null}`, then `data->'$.total_score_without_mods'` will return `CAST('null' AS JSON)` which IS NOT NULL.
                     // We want to be matching both of these cases. Thus, we use `JSON_VALUE`, which bypasses this footgun.
-                    "SELECT * FROM scores WHERE `id` BETWEEN @lastId AND (@lastId + @batchSize - 1) AND JSON_VALUE(`data`, '$.total_score_without_mods') IS NULL ORDER BY `id`",
+                    """
+                    SELECT * FROM scores
+                    WHERE `id` BETWEEN @lastId AND (@lastId + @batchSize - 1)
+                        AND `legacy_score_id` IS NULL
+                        AND JSON_LENGTH(`data`, '$.mods') > 0
+                        AND JSON_VALUE(`data`, '$.total_score_without_mods') IS NULL
+                    ORDER BY `id`
+                    """,
                     new
                     {
                         lastId,
                         batchSize = BatchSize,
                     })).ToArray();
 
-                if (scoresWithMissing.Length == 0)
+                if (scoresToPopulate.Length == 0)
                 {
                     if (lastId > await conn.QuerySingleAsync<ulong>("SELECT MAX(id) FROM scores"))
                     {
@@ -75,11 +92,15 @@ public async Task<int> OnExecuteAsync(CancellationToken cancellationToken)
                     continue;
                 }
 
-                uint[] beatmapIds = scoresWithMissing.Select(score => score.beatmap_id).Distinct().ToArray();
+                var beatmapIds = scoresToPopulate.Select(score => score.beatmap_id).ToHashSet();
                 var beatmapsById = (await conn.QueryAsync<Beatmap>(@"SELECT * FROM `osu_beatmaps` WHERE `beatmap_id` IN @ids", new { ids = beatmapIds }))
                     .ToDictionary(beatmap => beatmap.beatmap_id);
 
-                foreach (var score in scoresWithMissing)
+                var buildIds = scoresToPopulate.Select(score => score.build_id).Where(id => id != null).Cast<ushort>().ToHashSet();
+                var buildsById = (await conn.QueryAsync<Build>(@"SELECT * FROM `osu_builds` WHERE `build_id` IN @ids", new { ids = buildIds }))
+                    .ToDictionary(build => build.build_id);
+
+                foreach (var score in scoresToPopulate)
                 {
                     if (!beatmapsById.TryGetValue(score.beatmap_id, out var beatmap))
                     {
@@ -89,7 +110,13 @@ public async Task<int> OnExecuteAsync(CancellationToken cancellationToken)
 
                     score.beatmap = beatmap;
                     var scoreInfo = score.ToScoreInfo();
-                    LegacyScoreDecoder.PopulateTotalScoreWithoutMods(scoreInfo);
+                    if (score.build_id != null && buildsById.TryGetValue(score.build_id.Value, out var build))
+                        scoreInfo.ClientVersion = build.version;
+                    var ruleset = LegacyRulesetHelper.GetRulesetFromLegacyId(scoreInfo.RulesetID);
+
+                    var scoreMultiplierCalculator = ruleset.CreateScoreMultiplierCalculator(new ScoreMultiplierContext(beatmap.GetLegacyBeatmapConversionDifficultyInfo(), scoreInfo));
+                    double modMultiplier = scoreMultiplierCalculator.CalculateFor(scoreInfo.Mods);
+                    scoreInfo.TotalScoreWithoutMods = (long)Math.Round(scoreInfo.TotalScore / modMultiplier);
 
                     if (Verbose)
                         Console.WriteLine($"Updating score {score.id} to {scoreInfo.TotalScoreWithoutMods} (without mods) / {score.total_score} (with mods)");

osu.Server.Queues.ScoreStatisticsProcessor/Commands/Maintenance/RecalculateModMultipliersCommand.cs

@@ -11,7 +11,10 @@
 using JetBrains.Annotations;
 using McMaster.Extensions.CommandLineUtils;
 using MySqlConnector;
+using osu.Game.Database;
+using osu.Game.Rulesets.Scoring;
 using osu.Server.QueueProcessor;
+using osu.Server.Queues.ScoreStatisticsProcessor.Helpers;
 using osu.Server.Queues.ScoreStatisticsProcessor.Models;
 
 namespace osu.Server.Queues.ScoreStatisticsProcessor.Commands.Maintenance
@@ -37,7 +40,9 @@ public class RecalculateModMultipliersCommand
         public async Task<int> OnExecuteAsync(CancellationToken cancellationToken)
         {
             ulong lastId = StartId ?? 0;
-            ulong updatedScores = 0;
+
+            ulong skipped = 0;
+            ulong updated = 0;
 
             using var conn = await DatabaseAccess.GetConnectionAsync(cancellationToken);
 
@@ -78,36 +83,66 @@ public async Task<int> OnExecuteAsync(CancellationToken cancellationToken)
 
                 foreach (var score in scoresWithMods)
                 {
-                    score.beatmap = beatmapsById[score.beatmap_id];
-                    var scoreInfo = score.ToScoreInfo();
+                    string source = score.is_legacy_score ? "stable" : "lazer ";
 
-                    if (scoreInfo.TotalScoreWithoutMods == 0 && scoreInfo.TotalScore != 0)
+                    if (!beatmapsById.TryGetValue(score.beatmap_id, out var beatmap))
                     {
-                        throw new InvalidOperationException($"Score with ID {score.id} has {scoreInfo.TotalScore} total score but {scoreInfo.TotalScoreWithoutMods} total score without mods. "
-                                                            + $"This is likely to indicate that {nameof(scoreInfo.TotalScoreWithoutMods)} was not correctly backpopulated on all scores "
-                                                            + "(or there is a process pushing new scores that was not updated to populate the field).");
+                        Console.WriteLine($"[{score.id,11} {source}] Skipped due to missing beatmap");
+                        skipped++;
+                        continue;
                     }
 
-                    double multiplier = 1;
+                    score.beatmap = beatmap;
+                    var scoreInfo = score.ToScoreInfo();
+                    var difficultyInfo = beatmap.GetLegacyBeatmapConversionDifficultyInfo();
+                    var ruleset = LegacyRulesetHelper.GetRulesetFromLegacyId(score.ruleset_id);
+
+                    long oldTotalScore = score.total_score;
+                    long newTotalScore;
 
-                    foreach (var mod in scoreInfo.Mods)
-                        multiplier *= mod.ScoreMultiplier;
+                    if (score.is_legacy_score)
+                    {
+                        var scoringAttributes = getScoringAttributesFor(score, conn)?.ToAttributes();
+
+                        if (scoringAttributes == null)
+                        {
+                            Console.WriteLine($"[{score.id,11} {source}] Skipped due to missing scoring attributes");
+                            continue;
+                        }
+
+                        StandardisedScoreMigrationTools.UpdateFromLegacy(scoreInfo, ruleset, difficultyInfo, scoringAttributes.Value);
+                        newTotalScore = scoreInfo.TotalScore;
+                    }
+                    else
+                    {
+                        if (scoreInfo.TotalScoreWithoutMods == 0 && scoreInfo.TotalScore != 0)
+                        {
+                            Console.WriteLine($"[{score.id,11} {source}] Skipped due to missing total score without mods");
+                            continue;
+                        }
 
-                    long newTotalScore = (long)Math.Round(scoreInfo.TotalScoreWithoutMods * multiplier);
+                        var multiplierCalculator = ruleset.CreateScoreMultiplierCalculator(new ScoreMultiplierContext(scoreInfo.BeatmapInfo!.Difficulty));
+                        double multiplier = multiplierCalculator.CalculateFor(scoreInfo.Mods);
 
-                    if (newTotalScore == scoreInfo.TotalScore)
+                        newTotalScore = (long)Math.Round(scoreInfo.TotalScoreWithoutMods * multiplier);
+                    }
+
+                    if (newTotalScore == oldTotalScore)
+                    {
+                        Console.WriteLine($"[{score.id,11} {source}] Skipped due to no change in score");
                         continue;
+                    }
 
-                    Console.WriteLine($"Updating score {score.id}. Without mods: {scoreInfo.TotalScoreWithoutMods}. With mods: {scoreInfo.TotalScore} (old) -> {newTotalScore} (new)");
+                    Console.WriteLine($"[{score.id,11} {source}] Updating score: {oldTotalScore,8} (old) -> {newTotalScore,8} (new)");
 
                     sqlBuffer.Append($@"UPDATE `scores` SET `total_score` = {newTotalScore} WHERE `id` = {score.id};");
                     elasticItems.Add(new ElasticQueuePusher.ElasticScoreItem { ScoreId = (long?)score.id });
-                    updatedScores++;
+                    updated++;
                 }
 
                 lastId += (ulong)BatchSize;
 
-                Console.WriteLine($"Processed up to {lastId - 1} ({updatedScores} updated)");
+                Console.WriteLine($"Processed up to {lastId - 1} ({updated} updated, {skipped} skipped)");
 
                 flush(conn);
             }
@@ -117,6 +152,18 @@ public async Task<int> OnExecuteAsync(CancellationToken cancellationToken)
             return 0;
         }
 
+        private static BeatmapScoringAttributes? getScoringAttributesFor(SoloScore score, MySqlConnection conn)
+        {
+            BeatmapScoringAttributes? scoreAttributes = conn.QuerySingleOrDefault<BeatmapScoringAttributes>(
+                "SELECT * FROM osu_beatmap_scoring_attribs WHERE beatmap_id = @BeatmapId AND mode = @RulesetId", new
+                {
+                    BeatmapId = score.beatmap_id,
+                    RulesetId = score.ruleset_id,
+                });
+
+            return scoreAttributes;
+        }
+
         private void flush(MySqlConnection conn, bool force = false)
         {
             int bufferLength = sqlBuffer.Length;

osu.Server.Queues.ScoreStatisticsProcessor/Models/Build.cs

@@ -13,6 +13,7 @@ namespace osu.Server.Queues.ScoreStatisticsProcessor.Models
     public class Build
     {
         public int build_id { get; set; }
+        public string version { get; set; } = string.Empty;
         public bool allow_ranking { get; set; }
         public bool allow_performance { get; set; }
     }