champions-of-luneraria/Assets/Luneraria/Tests/Core/GoldenSuite.Compare.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;

namespace Luneraria.Core.Tests
{
    /// <summary>
    /// Golden replays: every scenario of Golden/scenarios.json runs on the C# core and each
    /// snapshot is compared with the Godot reference (Golden/golden.json) field by field.
    /// Integers, strings, flags, ids, ownership, purchases, targets and results must be equal;
    /// numbers that pass through floating point use the tolerances below (docs/port/U1-RULES.md).
    /// The largest deviation per category is noted in the report, and per domain (gold, hit
    /// points, positions, towers, Ancients, waves, heroes, boss, winner, ...) for the port QA.
    /// </summary>
    public sealed partial class GoldenSuite
    {
        /// <summary>Tolerance categories: hit points (absolute, 0.01 HP or 1e-9 relative), positions and lane progress (1 mm), angles, times, effects.</summary>
        static readonly Dictionary<string, double> Tolerances = new Dictionary<string, double>
        {
            { "hp", 0.01 }, { "position", 0.001 }, { "angle", 0.0001 }, { "time", 0.000001 }, { "effect", 0.000001 }, { "other", 0.000001 },
        };

        readonly Dictionary<string, double> _worst = new Dictionary<string, double>();
        readonly Dictionary<string, int> _counts = new Dictionary<string, int>();
        int _compared;

        sealed class DomainStat
        {
            public int Values;
            public int Differences;
            public double Worst;
        }

        readonly OrderedMap<string, DomainStat> _domains = new OrderedMap<string, DomainStat>();
        List<BuildingDef> _buildings;

        partial void RunScenarios()
        {
            var spec = MiniJson.Obj(MiniJson.Parse(File.ReadAllText(Path.Combine(GoldenDirectory, "scenarios.json"))));
            var golden = MiniJson.Obj(MiniJson.Obj(MiniJson.Parse(File.ReadAllText(Path.Combine(GoldenDirectory, "golden.json"))))["scenarios"]);
            foreach (object item in MiniJson.Arr(spec["scenarios"]))
            {
                var scenario = MiniJson.Obj(item);
                string id = MiniJson.Str(scenario["id"]);
                object reference;
                if (!golden.TryGetValue(id, out reference))
                {
                    Check(false, id + ": no golden reference (run tools/godot-golden/run_golden.sh)");
                    continue;
                }
                var replay = new GoldenReplay();
                foreach (object op in MiniJson.Arr(scenario["ops"]))
                    replay.Apply(MiniJson.Obj(op));
                var expected = MiniJson.Obj(reference);
                List<object> expectedSnapshots = MiniJson.Arr(expected["snapshots"]);
                Check(expectedSnapshots.Count == replay.Snapshots.Count, string.Format("{0}: {1} snapshots like the reference ({2})", id, replay.Snapshots.Count, expectedSnapshots.Count));
                for (int i = 0; i < Math.Min(expectedSnapshots.Count, replay.Snapshots.Count); i++)
                {
                    var problems = new List<string>();
                    string label = MiniJson.Str(MiniJson.Obj(expectedSnapshots[i])["label"]);
                    CompareValue(id + "/" + label, "", expectedSnapshots[i], replay.Snapshots[i], problems);
                    Check(problems.Count == 0, id + "/" + label + " matches the reference" + (problems.Count > 0 ? ": " + problems.Count + " differences, first " + string.Join("; ", problems.Take(6)) : ""));
                }
                var results = new List<string>();
                CompareValue(id + "/results", "results", expected["results"], replay.Results, results);
                Check(results.Count == 0, id + ": command results and messages match the reference" + (results.Count > 0 ? ": " + string.Join("; ", results.Take(6)) : ""));
            }
            Note(string.Format("golden values compared: {0}", _compared));
            foreach (KeyValuePair<string, double> pair in _worst.OrderBy(p => p.Key))
                Note(string.Format("largest deviation {0}: {1:E2} over {2} values (tolerance {3:E0})", pair.Key, pair.Value, _counts[pair.Key], Tolerances[pair.Key]));
            foreach (KeyValuePair<string, DomainStat> pair in _domains.OrderBy(p => p.Key, StringComparer.Ordinal))
                Note(string.Format("domain {0}: {1} values, {2} differences, largest numeric deviation {3:E2}", pair.Key, pair.Value.Values, pair.Value.Differences, pair.Value.Worst));
        }

        /// <summary>Domain of a compared value for the report, from its path in the snapshot.</summary>
        string Domain(string path)
        {
            if (path.Contains("/results"))
                return "commands";
            int dot = path.IndexOf('.');
            if (dot < 0)
                return "other";
            string[] parts = path.Substring(dot + 1).Split('.');
            string top = Head(parts[0]);
            string field = parts.Length > 1 ? Head(parts[1]) : "";
            switch (top)
            {
                case "winner":
                case "end_note":
                case "phase":
                    return "winner";
                case "round":
                    return "rounds";
                case "players":
                    if (field == "gold" || field == "draft_gold" || field == "earned" || field == "spent")
                        return "gold";
                    if (field == "hero" || field == "draft_hero")
                        return "heroes";
                    if (field == "boss_lane")
                        return "boss";
                    return "purchases";
                case "buildings":
                    int open = parts[0].IndexOf('[');
                    int index = open < 0 ? -1 : int.Parse(parts[0].Substring(open + 1, parts[0].IndexOf(']') - open - 1));
                    if (_buildings == null)
                        _buildings = new LuneMap(new LuneBalance()).Buildings;
                    return index >= 0 && index < _buildings.Count && _buildings[index].Tier == "core" ? "ancients" : "towers";
                case "boss":
                case "boss_log":
                    return "boss";
                case "announcements":
                    return "announcements";
                case "settlement":
                    if (field == "gains" || field == "income")
                        return "gold";
                    if (field == "xp")
                        return "heroes";
                    if (field == "lost")
                        return "towers";
                    if (field == "end_reason")
                        return "winner";
                    if (field == "boss")
                        return "boss";
                    if (field == "lane_kills")
                        return "waves";
                    return "combat";
                case "combat":
                    if (field == "destroyed")
                        return "ancients";
                    if (field.StartsWith("grotto", StringComparison.Ordinal))
                        return "boss";
                    if (field == "end_reason" || field == "finished")
                        return "winner";
                    if (field == "mobile")
                        return "waves";
                    return "combat";
                case "units":
                    if (field == "x" || field == "y" || field == "progress")
                        return "positions";
                    if (field == "facing")
                        return "facing";
                    if (field == "hp" || field == "max_hp" || field == "shield")
                        return "hp";
                    if (field == "target")
                        return "targets";
                    return "units";
                case "events":
                    return "events";
                case "wave":
                    return "waves";
            }
            return "other";
        }

        static string Head(string segment)
        {
            int bracket = segment.IndexOf('[');
            return bracket < 0 ? segment : segment.Substring(0, bracket);
        }

        void Track(string path, bool different, double deviation)
        {
            string domain = Domain(path);
            DomainStat stat;
            if (!_domains.TryGetValue(domain, out stat))
            {
                stat = new DomainStat();
                _domains[domain] = stat;
            }
            stat.Values++;
            if (different)
                stat.Differences++;
            if (deviation > stat.Worst)
                stat.Worst = deviation;
        }

        static string Category(string key)
        {
            switch (key)
            {
                case "hp":
                case "max_hp":
                case "shield":
                case "grotto_boss_hp":
                case "buildings":
                    return "hp";
                case "x":
                case "y":
                case "progress":
                    return "position";
                case "facing":
                    return "angle";
                case "elapsed":
                case "combat_time":
                case "events":
                    return "time";
                case "effects":
                case "dots":
                    return "effect";
            }
            return "other";
        }

        void CompareValue(string path, string key, object expected, object actual, List<string> problems)
        {
            if (expected == null || actual == null)
            {
                if (expected != actual)
                    problems.Add(path + ": " + Show(expected) + " != " + Show(actual));
                return;
            }
            if (IsNumber(expected) && IsNumber(actual))
            {
                _compared++;
                if (expected is long el && actual is long al)
                {
                    if (el != al)
                        problems.Add(path + ": " + el + " != " + al);
                    Track(path, el != al, Math.Abs((double)el - al));
                    return;
                }
                double e = MiniJson.Num(expected);
                double a = MiniJson.Num(actual);
                string category = Category(key);
                double deviation = Math.Abs(e - a);
                double tolerance = Math.Max(Tolerances[category], 1e-9 * Math.Abs(e));
                double worst;
                if (!_worst.TryGetValue(category, out worst) || deviation > worst)
                    _worst[category] = deviation;
                int count;
                _counts.TryGetValue(category, out count);
                _counts[category] = count + 1;
                bool outside = deviation > tolerance || double.IsNaN(a) != double.IsNaN(e);
                if (outside)
                    problems.Add(string.Format("{0}: {1:R} != {2:R}", path, e, a));
                Track(path, outside, deviation);
                return;
            }
            if (expected is OrderedMap<string, object> em && actual is OrderedMap<string, object> am)
            {
                if (!em.Keys.SequenceEqual(am.Keys))
                    problems.Add(path + ": keys " + string.Join(",", em.Keys) + " != " + string.Join(",", am.Keys));
                foreach (KeyValuePair<string, object> pair in em)
                {
                    object other;
                    if (am.TryGetValue(pair.Key, out other))
                        CompareValue(path + "." + pair.Key, pair.Key, pair.Value, other, problems);
                }
                return;
            }
            if (expected is List<object> list && actual is List<object> otherList)
            {
                if (list.Count != otherList.Count)
                {
                    problems.Add(path + ": " + list.Count + " entries != " + otherList.Count);
                    return;
                }
                for (int i = 0; i < list.Count; i++)
                    CompareValue(path + "[" + i + "]", key, list[i], otherList[i], problems);
                return;
            }
            _compared++;
            bool differs = !Equals(Normalize(expected), Normalize(actual));
            if (differs)
                problems.Add(path + ": " + Show(expected) + " != " + Show(actual));
            Track(path, differs, 0.0);
        }

        static bool IsNumber(object value) { return value is long || value is double; }

        static object Normalize(object value) { return value is bool b ? (object)b : MiniJson.Str(value); }

        static string Show(object value) { return value == null ? "null" : "'" + MiniJson.Str(value) + "'"; }
    }
}