From e79e796a21b25b795e2f025b96aeb005d90d62d9 Mon Sep 17 00:00:00 2001 From: CroakFang <1605296300@qq.com> Date: Thu, 15 Dec 2022 13:55:32 +0800 Subject: [PATCH] Generate Meta database from game server --- .gitignore | 1 + Assets/Scripts/Manifest.cs | 6 - Assets/Scripts/Manifest.cs.meta | 11 -- Assets/Scripts/Manifest/BSVReader.cs | 8 +- Assets/Scripts/Manifest/ManifestDB.cs | 212 ++++++++++++++++++++- Assets/Scripts/Manifest/ManifestEntry.cs | 31 +++ Assets/Scripts/UmaDatabase/UmaDataTypes.cs | 3 +- 7 files changed, 245 insertions(+), 27 deletions(-) delete mode 100644 Assets/Scripts/Manifest.cs delete mode 100644 Assets/Scripts/Manifest.cs.meta diff --git a/.gitignore b/.gitignore index 7cb2e66..cf16bff 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,4 @@ crashlytics-build.properties /Assets/Plugins/RootMotion/PuppetMaster/_Integration /VMDRecords +/umamusume diff --git a/Assets/Scripts/Manifest.cs b/Assets/Scripts/Manifest.cs deleted file mode 100644 index 02d7b8f..0000000 --- a/Assets/Scripts/Manifest.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Assets.Scripts -{ - internal class Manifest - { - } -} \ No newline at end of file diff --git a/Assets/Scripts/Manifest.cs.meta b/Assets/Scripts/Manifest.cs.meta deleted file mode 100644 index 9d555b1..0000000 --- a/Assets/Scripts/Manifest.cs.meta +++ /dev/null @@ -1,11 +0,0 @@ -fileFormatVersion: 2 -guid: 6edaa1ece12a29d43beec62ef0d73d29 -MonoImporter: - externalObjects: {} - serializedVersion: 2 - defaultReferences: [] - executionOrder: 0 - icon: {instanceID: 0} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Assets/Scripts/Manifest/BSVReader.cs b/Assets/Scripts/Manifest/BSVReader.cs index 0e460fd..2ead37c 100644 --- a/Assets/Scripts/Manifest/BSVReader.cs +++ b/Assets/Scripts/Manifest/BSVReader.cs @@ -1,6 +1,7 @@ using K4os.Compression.LZ4.Streams; using System; using System.IO; +using UnityEngine; public class BSVReader { @@ -67,7 +68,7 @@ public class BSVReader public static ArraySegment ReadText(byte[] buf, ref int offset) { int start = offset; - while (buf[offset] != 0 && offset < buf.Length) + while (offset < buf.Length &&buf[offset] != 0) { offset++; } @@ -102,7 +103,7 @@ public class BSVReader return subArray; } - internal static IBSVReader Init(string path, ReadMode mode, bool isCompressed) + internal static IBSVReader Init(string path, bool isCompressed,ReadMode mode = ReadMode.Memory) { try { @@ -131,7 +132,8 @@ public class BSVReader } catch (Exception e) { - throw e; + Debug.LogError("Read Manifest File Error :" + e); + return null; } } diff --git a/Assets/Scripts/Manifest/ManifestDB.cs b/Assets/Scripts/Manifest/ManifestDB.cs index d61ad03..86888d5 100644 --- a/Assets/Scripts/Manifest/ManifestDB.cs +++ b/Assets/Scripts/Manifest/ManifestDB.cs @@ -1,26 +1,160 @@ using Mono.Data.Sqlite; using System; +using System.Collections; +using System.IO; +using System.Text; using UnityEngine; +using UnityEngine.Networking; using static BSVReader; using static ManifestCategory; public class ManifestDB { - public SqliteConnection metaDb; + public SqliteConnection MetaDB; + public string DBPath; + public static string MANIFEST_ROOT_URL = "https://prd-storage-app-umamusume.akamaized.net/dl/resources/Manifest"; + public ManifestDB(string DBPath) + { + this.DBPath = DBPath; + if (File.Exists(DBPath)) + { + MetaDB = new SqliteConnection($@"Data Source={DBPath}"); + } + else + { + SqliteConnection.CreateFile(DBPath); + MetaDB = new SqliteConnection($@"Data Source={DBPath}"); + MetaDB.Open(); + SqliteCommand cmd = new SqliteCommand( + "CREATE TABLE `a` ( `i` INTEGER PRIMARY KEY,`n` TEXT NOT NULL,`d` TEXT,`g` INTEGER(4) NOT NULL,`l` INTEGER(8) NOT NULL,`c` INTEGER(8) NOT NULL,`h` TEXT NOT NULL,`m` TEXT NOT NULL,`k` INTEGER(1) NOT NULL,`s` INTEGER(1) NOT NULL,`p` INTEGER(4) NOT NULL DEFAULT 0)"); + cmd.Connection = MetaDB; + cmd.ExecuteNonQuery(); + cmd.CommandText = "CREATE INDEX `a0` ON `a`(`n`, `s`)"; + cmd.ExecuteNonQuery(); + cmd.CommandText = "CREATE INDEX `a2` ON `a`(`m`, `s`)"; + cmd.ExecuteNonQuery(); + cmd.CommandText = "CREATE INDEX `a3` ON `a`(`g`, 's')"; + cmd.ExecuteNonQuery(); + cmd.CommandText = "CREATE INDEX `a4` ON `a`(`p`)"; + cmd.ExecuteNonQuery(); + cmd.CommandText = "CREATE INDEX `a1` ON `a`(`s`)"; + cmd.ExecuteNonQuery(); + } + } + + + //umamusume uses the manifest file to generate meta database. + //it needs to get the root manifest file first + //then read the root manifest to get other manifest file information + //recursively read the manifest file to obtain all resource files information + //finally, insert all data into the meta database + public IEnumerator GenerateMetaDB(ulong rootSize, ulong rootCheckSum) + { + if (MetaDB == null || MetaDB.State == System.Data.ConnectionState.Closed) { + Debug.LogError("Open Meta Database Error"); + yield break; + } + + //Insert root + ManifestEntry rootEntry = new ManifestEntry(Encoding.UTF8.GetBytes("root"), null, 0, 0, rootSize, rootCheckSum, "", 3); + rootEntry.hname = rootEntry.CalHameString(); + + SqliteCommand command = new SqliteCommand(MetaDB); + command.CommandText = "DELETE FROM a"; + command.ExecuteNonQuery(); + + int index = 1; + InsertManifestEntry(index, "manifest3", rootEntry, ref command); + + //Insert platform + ManifestEntry[] platformEntrys = null; + yield return GetManifest(rootEntry.hname, Kind.PlatformManifest, entrys => { platformEntrys = entrys; }); + if (platformEntrys == null) + { + Debug.LogError("Read PlatformManifest Error"); + MetaDB.Close(); + yield break; + } + ManifestEntry platformEntry = GetLocalPlatformEntry(platformEntrys); + index++; + InsertManifestEntry(index, "manifest2", platformEntry, ref command); + + //Insert AssetManifest + ManifestEntry[] assetEntrys = null; + yield return GetManifest(platformEntry.hname, Kind.AssetManifest, entrys => { assetEntrys = entrys; }); + if (assetEntrys == null) + { + Debug.LogError("Read AssetManifest Error"); + MetaDB.Close(); + yield break; + } + using (SqliteTransaction tran = MetaDB.BeginTransaction()) + { + foreach (var entry in assetEntrys) + { + index++; + InsertManifestEntry(index, "manifest", entry, ref command); + } + tran.Commit(); + } + + //Insert Asset + foreach (var assetEntry in assetEntrys) + { + ManifestEntry[] assets = null; + yield return GetManifest(assetEntry.hname, Kind.Default, entrys => { assets = entrys; }); + if (assets == null) + { + Debug.LogError("Read AssetManifest Error"); + continue; + } + using (SqliteTransaction tran = MetaDB.BeginTransaction()) + { + foreach (var entry in assets) + { + index++; + InsertManifestEntry(index, assetEntry.tname, entry, ref command); + } + tran.Commit(); + } + } + MetaDB.Close(); + } + + private void InsertManifestEntry(int index, string type, ManifestEntry entry, ref SqliteCommand command) + { + command.CommandText = "INSERT INTO a (i,n,d,g,l,c,h,m,k,s,p) VALUES (@id,@name,@deps,@group,@length,@check,@hash,@m,@k,@s,@p)"; + command.Parameters.Clear(); + command.Parameters.Add(new SqliteParameter("@id", index)); + command.Parameters.Add(new SqliteParameter("@name", (type.Contains("manifest") ? "//" : "") + entry.tname)); + command.Parameters.Add(new SqliteParameter("@deps", entry.tdeps)); + command.Parameters.Add(new SqliteParameter("@group", entry.group)); + command.Parameters.Add(new SqliteParameter("@length", (long)entry.size)); + command.Parameters.Add(new SqliteParameter("@check", (long)entry.checksum)); + command.Parameters.Add(new SqliteParameter("@hash", entry.hname)); + command.Parameters.Add(new SqliteParameter("@m", type)); + command.Parameters.Add(new SqliteParameter("@k", (byte)entry.kind)); + command.Parameters.Add(new SqliteParameter("@s", 1)); + command.Parameters.Add(new SqliteParameter("@p", entry.priority)); + command.ExecuteNonQuery(); + } public static ManifestEntry[] GetManifest(string path, bool isCompressed, Kind kind) { - IBSVReader reader = Init(path, ReadMode.Memory, isCompressed); + IBSVReader reader = Init(path, isCompressed); + if (reader == null) return null; + ulong count = reader.GetRowCount(); ManifestEntry[] manifests = new ManifestEntry[count]; - if (kind == Kind.AssetManifest) + if (kind == Kind.Default) { for (ulong i = 0; i < count; i++) { manifests[i].kind = (int)kind; if (!reader.ReadLine(ManifestEntry.GetBsvParser(ManifestEntry.Format.Full, reader), ref manifests[i])) { - throw new Exception("ReadLineError"); + Debug.LogError("ReadLineError"); + return null; } } } @@ -31,13 +165,79 @@ public class ManifestDB manifests[i].kind = (int)kind; if (!reader.ReadLine(ManifestEntry.GetBsvParser(ManifestEntry.Format.Simplified, reader), ref manifests[i])) { - throw new Exception("ReadLineError"); + Debug.LogError("ReadLineError"); + return null; } } } return manifests; + } - + public IEnumerator GetManifest(string hash, Kind kind, Action callback) + { + var path = GetManifestPath(hash); + if (!File.Exists(path)) + { + yield return DownloadManifest(hash); + } + if (File.Exists(path)) + { + callback.Invoke(GetManifest(path, true, kind)); + } + } + + public IEnumerator DownloadManifest(string hash) + { + UnityWebRequest www = UnityWebRequest.Get(GetManifestRequestUrl(hash)); + yield return www.SendWebRequest(); + if (www.result == UnityWebRequest.Result.Success) + { + File.WriteAllBytes(GetManifestPath(hash), www.downloadHandler.data); + } + else + { + Debug.LogError("Download Manifest Failed :" + www.error); + } + } + + public string GetManifestPath(string hash, bool createPath = true) + { + string path = $"{Path.GetDirectoryName(DBPath)}/dat/{hash.Substring(0, 2)}"; + if (createPath && !Directory.Exists(path)) + { + Directory.CreateDirectory(path); + } + return path + $"/{hash}"; + } + + public static string GetManifestRequestUrl(string hash) + { + return $"{MANIFEST_ROOT_URL}/{hash.Substring(0, 2)}/{hash}"; + } + + public static ManifestEntry GetLocalPlatformEntry(ManifestEntry[] platformEntrys) + { + foreach (var entry in platformEntrys) + { + if (Encoding.UTF8.GetString(entry.name) == "Windows" && Application.platform == RuntimePlatform.WindowsPlayer) + { + return entry; + } + else if (Encoding.UTF8.GetString(entry.name) == "IOS" && Application.platform == RuntimePlatform.IPhonePlayer) + { + return entry; + } + else if (Encoding.UTF8.GetString(entry.name) == "Android" && Application.platform == RuntimePlatform.Android) + { + return entry; + } + else + { + return entry; + } + } + return platformEntrys[0]; + } } diff --git a/Assets/Scripts/Manifest/ManifestEntry.cs b/Assets/Scripts/Manifest/ManifestEntry.cs index 8ed0730..c522d81 100644 --- a/Assets/Scripts/Manifest/ManifestEntry.cs +++ b/Assets/Scripts/Manifest/ManifestEntry.cs @@ -9,6 +9,8 @@ public struct ManifestEntry { public byte[] name; public byte[] deps; + public string tname; + public string tdeps; public uint group; public uint priority; public ulong size; @@ -16,12 +18,27 @@ public struct ManifestEntry public string hname; public int kind; + public ManifestEntry(byte[] name, byte[] deps, uint group, uint priority, ulong size, ulong checksum, string hname, int kind) + { + this.name = name; + this.deps = deps; + this.group = group; + this.priority = priority; + this.size = size; + this.checksum = checksum; + this.hname = hname; + this.kind = kind; + this.tname = Encoding.UTF8.GetString(this.name); + this.tdeps = (this.deps == null ? null : Encoding.UTF8.GetString(this.deps)); + } + public enum Format { Simplified = 0, Full = 1 } + public static byte[] CalHame(ulong checksum, ulong size, byte[] name) { if (name == null) return null; @@ -37,6 +54,7 @@ public struct ManifestEntry return sha1.ComputeHash(allbyte); } + public static string CalHameString(ulong checksum, ulong size, byte[] name) { var bytes = CalHame(checksum, size, name); @@ -49,6 +67,11 @@ public struct ManifestEntry return CalHameString(checksum, size, Encoding.UTF8.GetBytes(name)); } + public string CalHameString() + { + return CalHameString(checksum, size, name); + } + public override string ToString() { StringBuilder s = new StringBuilder(); @@ -90,6 +113,8 @@ public struct ManifestEntry dat.name = GetArray(ReadText(buf, ref offset)); dat.size = ReadVLQ(buf, ref offset); dat.checksum = ReadUNum(buf, ref offset, 8); + dat.hname = CalHameString(dat.checksum, dat.size, dat.name); + dat.tname = Encoding.UTF8.GetString(dat.name); } } @@ -105,6 +130,9 @@ public struct ManifestEntry dat.priority = (uint)ReadVLQ(buf, ref offset, 4); dat.size = (uint)ReadVLQ(buf, ref offset, 8); dat.checksum = ReadUNum(buf, ref offset, 8); + dat.hname = CalHameString(dat.checksum, dat.size, dat.name); + dat.tname = Encoding.UTF8.GetString(dat.name); + dat.tdeps = (dat.deps == null ? null : Encoding.UTF8.GetString(dat.deps)); } } @@ -118,6 +146,9 @@ public struct ManifestEntry dat.group = (uint)ReadVLQ(buf, ref offset, 4); dat.size = (uint)ReadVLQ(buf, ref offset, 8); dat.checksum = ReadUNum(buf, ref offset, 8); + dat.hname = CalHameString(dat.checksum, dat.size, dat.name); + dat.tname = Encoding.UTF8.GetString(dat.name); + dat.tdeps = (dat.deps == null ? null : Encoding.UTF8.GetString(dat.deps)); dat.priority = 0; } } diff --git a/Assets/Scripts/UmaDatabase/UmaDataTypes.cs b/Assets/Scripts/UmaDatabase/UmaDataTypes.cs index 80afe32..3ba085f 100644 --- a/Assets/Scripts/UmaDatabase/UmaDataTypes.cs +++ b/Assets/Scripts/UmaDatabase/UmaDataTypes.cs @@ -128,5 +128,6 @@ public enum UmaFileType uianimation, transferevent, teambuilding, - challengematch + challengematch, + collectevent }