Generate Meta database from game server

This commit is contained in:
CroakFang
2022-12-15 13:55:32 +08:00
parent 0d22399fcb
commit e79e796a21
7 changed files with 245 additions and 27 deletions
-6
View File
@@ -1,6 +0,0 @@
namespace Assets.Scripts
{
internal class Manifest
{
}
}
-11
View File
@@ -1,11 +0,0 @@
fileFormatVersion: 2
guid: 6edaa1ece12a29d43beec62ef0d73d29
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:
+5 -3
View File
@@ -1,6 +1,7 @@
using K4os.Compression.LZ4.Streams;
using System;
using System.IO;
using UnityEngine;
public class BSVReader
{
@@ -67,7 +68,7 @@ public class BSVReader
public static ArraySegment<byte> ReadText(byte[] buf, ref int offset)
{
int start = offset;
while (buf[offset] != 0 && offset < buf.Length)
while (offset < buf.Length &&buf[offset] != 0)
{
offset++;
}
@@ -102,7 +103,7 @@ public class BSVReader
return subArray;
}
internal static IBSVReader Init(string path, ReadMode mode, bool isCompressed)
internal static IBSVReader Init(string path, bool isCompressed,ReadMode mode = ReadMode.Memory)
{
try
{
@@ -131,7 +132,8 @@ public class BSVReader
}
catch (Exception e)
{
throw e;
Debug.LogError("Read Manifest File Error :" + e);
return null;
}
}
+206 -6
View File
@@ -1,26 +1,160 @@
using Mono.Data.Sqlite;
using System;
using System.Collections;
using System.IO;
using System.Text;
using UnityEngine;
using UnityEngine.Networking;
using static BSVReader;
using static ManifestCategory;
public class ManifestDB
{
public SqliteConnection metaDb;
public SqliteConnection MetaDB;
public string DBPath;
public static string MANIFEST_ROOT_URL = "https://prd-storage-app-umamusume.akamaized.net/dl/resources/Manifest";
public ManifestDB(string DBPath)
{
this.DBPath = DBPath;
if (File.Exists(DBPath))
{
MetaDB = new SqliteConnection($@"Data Source={DBPath}");
}
else
{
SqliteConnection.CreateFile(DBPath);
MetaDB = new SqliteConnection($@"Data Source={DBPath}");
MetaDB.Open();
SqliteCommand cmd = new SqliteCommand(
"CREATE TABLE `a` ( `i` INTEGER PRIMARY KEY,`n` TEXT NOT NULL,`d` TEXT,`g` INTEGER(4) NOT NULL,`l` INTEGER(8) NOT NULL,`c` INTEGER(8) NOT NULL,`h` TEXT NOT NULL,`m` TEXT NOT NULL,`k` INTEGER(1) NOT NULL,`s` INTEGER(1) NOT NULL,`p` INTEGER(4) NOT NULL DEFAULT 0)");
cmd.Connection = MetaDB;
cmd.ExecuteNonQuery();
cmd.CommandText = "CREATE INDEX `a0` ON `a`(`n`, `s`)";
cmd.ExecuteNonQuery();
cmd.CommandText = "CREATE INDEX `a2` ON `a`(`m`, `s`)";
cmd.ExecuteNonQuery();
cmd.CommandText = "CREATE INDEX `a3` ON `a`(`g`, 's')";
cmd.ExecuteNonQuery();
cmd.CommandText = "CREATE INDEX `a4` ON `a`(`p`)";
cmd.ExecuteNonQuery();
cmd.CommandText = "CREATE INDEX `a1` ON `a`(`s`)";
cmd.ExecuteNonQuery();
}
}
//umamusume uses the manifest file to generate meta database.
//it needs to get the root manifest file first
//then read the root manifest to get other manifest file information
//recursively read the manifest file to obtain all resource files information
//finally, insert all data into the meta database
public IEnumerator GenerateMetaDB(ulong rootSize, ulong rootCheckSum)
{
if (MetaDB == null || MetaDB.State == System.Data.ConnectionState.Closed) {
Debug.LogError("Open Meta Database Error");
yield break;
}
//Insert root
ManifestEntry rootEntry = new ManifestEntry(Encoding.UTF8.GetBytes("root"), null, 0, 0, rootSize, rootCheckSum, "", 3);
rootEntry.hname = rootEntry.CalHameString();
SqliteCommand command = new SqliteCommand(MetaDB);
command.CommandText = "DELETE FROM a";
command.ExecuteNonQuery();
int index = 1;
InsertManifestEntry(index, "manifest3", rootEntry, ref command);
//Insert platform
ManifestEntry[] platformEntrys = null;
yield return GetManifest(rootEntry.hname, Kind.PlatformManifest, entrys => { platformEntrys = entrys; });
if (platformEntrys == null)
{
Debug.LogError("Read PlatformManifest Error");
MetaDB.Close();
yield break;
}
ManifestEntry platformEntry = GetLocalPlatformEntry(platformEntrys);
index++;
InsertManifestEntry(index, "manifest2", platformEntry, ref command);
//Insert AssetManifest
ManifestEntry[] assetEntrys = null;
yield return GetManifest(platformEntry.hname, Kind.AssetManifest, entrys => { assetEntrys = entrys; });
if (assetEntrys == null)
{
Debug.LogError("Read AssetManifest Error");
MetaDB.Close();
yield break;
}
using (SqliteTransaction tran = MetaDB.BeginTransaction())
{
foreach (var entry in assetEntrys)
{
index++;
InsertManifestEntry(index, "manifest", entry, ref command);
}
tran.Commit();
}
//Insert Asset
foreach (var assetEntry in assetEntrys)
{
ManifestEntry[] assets = null;
yield return GetManifest(assetEntry.hname, Kind.Default, entrys => { assets = entrys; });
if (assets == null)
{
Debug.LogError("Read AssetManifest Error");
continue;
}
using (SqliteTransaction tran = MetaDB.BeginTransaction())
{
foreach (var entry in assets)
{
index++;
InsertManifestEntry(index, assetEntry.tname, entry, ref command);
}
tran.Commit();
}
}
MetaDB.Close();
}
private void InsertManifestEntry(int index, string type, ManifestEntry entry, ref SqliteCommand command)
{
command.CommandText = "INSERT INTO a (i,n,d,g,l,c,h,m,k,s,p) VALUES (@id,@name,@deps,@group,@length,@check,@hash,@m,@k,@s,@p)";
command.Parameters.Clear();
command.Parameters.Add(new SqliteParameter("@id", index));
command.Parameters.Add(new SqliteParameter("@name", (type.Contains("manifest") ? "//" : "") + entry.tname));
command.Parameters.Add(new SqliteParameter("@deps", entry.tdeps));
command.Parameters.Add(new SqliteParameter("@group", entry.group));
command.Parameters.Add(new SqliteParameter("@length", (long)entry.size));
command.Parameters.Add(new SqliteParameter("@check", (long)entry.checksum));
command.Parameters.Add(new SqliteParameter("@hash", entry.hname));
command.Parameters.Add(new SqliteParameter("@m", type));
command.Parameters.Add(new SqliteParameter("@k", (byte)entry.kind));
command.Parameters.Add(new SqliteParameter("@s", 1));
command.Parameters.Add(new SqliteParameter("@p", entry.priority));
command.ExecuteNonQuery();
}
public static ManifestEntry[] GetManifest(string path, bool isCompressed, Kind kind)
{
IBSVReader reader = Init(path, ReadMode.Memory, isCompressed);
IBSVReader reader = Init(path, isCompressed);
if (reader == null) return null;
ulong count = reader.GetRowCount();
ManifestEntry[] manifests = new ManifestEntry[count];
if (kind == Kind.AssetManifest)
if (kind == Kind.Default)
{
for (ulong i = 0; i < count; i++)
{
manifests[i].kind = (int)kind;
if (!reader.ReadLine(ManifestEntry.GetBsvParser(ManifestEntry.Format.Full, reader), ref manifests[i]))
{
throw new Exception("ReadLineError");
Debug.LogError("ReadLineError");
return null;
}
}
}
@@ -31,13 +165,79 @@ public class ManifestDB
manifests[i].kind = (int)kind;
if (!reader.ReadLine(ManifestEntry.GetBsvParser(ManifestEntry.Format.Simplified, reader), ref manifests[i]))
{
throw new Exception("ReadLineError");
Debug.LogError("ReadLineError");
return null;
}
}
}
return manifests;
}
public IEnumerator GetManifest(string hash, Kind kind, Action<ManifestEntry[]> callback)
{
var path = GetManifestPath(hash);
if (!File.Exists(path))
{
yield return DownloadManifest(hash);
}
if (File.Exists(path))
{
callback.Invoke(GetManifest(path, true, kind));
}
}
public IEnumerator DownloadManifest(string hash)
{
UnityWebRequest www = UnityWebRequest.Get(GetManifestRequestUrl(hash));
yield return www.SendWebRequest();
if (www.result == UnityWebRequest.Result.Success)
{
File.WriteAllBytes(GetManifestPath(hash), www.downloadHandler.data);
}
else
{
Debug.LogError("Download Manifest Failed :" + www.error);
}
}
public string GetManifestPath(string hash, bool createPath = true)
{
string path = $"{Path.GetDirectoryName(DBPath)}/dat/{hash.Substring(0, 2)}";
if (createPath && !Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
return path + $"/{hash}";
}
public static string GetManifestRequestUrl(string hash)
{
return $"{MANIFEST_ROOT_URL}/{hash.Substring(0, 2)}/{hash}";
}
public static ManifestEntry GetLocalPlatformEntry(ManifestEntry[] platformEntrys)
{
foreach (var entry in platformEntrys)
{
if (Encoding.UTF8.GetString(entry.name) == "Windows" && Application.platform == RuntimePlatform.WindowsPlayer)
{
return entry;
}
else if (Encoding.UTF8.GetString(entry.name) == "IOS" && Application.platform == RuntimePlatform.IPhonePlayer)
{
return entry;
}
else if (Encoding.UTF8.GetString(entry.name) == "Android" && Application.platform == RuntimePlatform.Android)
{
return entry;
}
else
{
return entry;
}
}
return platformEntrys[0];
}
}
+31
View File
@@ -9,6 +9,8 @@ public struct ManifestEntry
{
public byte[] name;
public byte[] deps;
public string tname;
public string tdeps;
public uint group;
public uint priority;
public ulong size;
@@ -16,12 +18,27 @@ public struct ManifestEntry
public string hname;
public int kind;
public ManifestEntry(byte[] name, byte[] deps, uint group, uint priority, ulong size, ulong checksum, string hname, int kind)
{
this.name = name;
this.deps = deps;
this.group = group;
this.priority = priority;
this.size = size;
this.checksum = checksum;
this.hname = hname;
this.kind = kind;
this.tname = Encoding.UTF8.GetString(this.name);
this.tdeps = (this.deps == null ? null : Encoding.UTF8.GetString(this.deps));
}
public enum Format
{
Simplified = 0,
Full = 1
}
public static byte[] CalHame(ulong checksum, ulong size, byte[] name)
{
if (name == null) return null;
@@ -37,6 +54,7 @@ public struct ManifestEntry
return sha1.ComputeHash(allbyte);
}
public static string CalHameString(ulong checksum, ulong size, byte[] name)
{
var bytes = CalHame(checksum, size, name);
@@ -49,6 +67,11 @@ public struct ManifestEntry
return CalHameString(checksum, size, Encoding.UTF8.GetBytes(name));
}
public string CalHameString()
{
return CalHameString(checksum, size, name);
}
public override string ToString()
{
StringBuilder s = new StringBuilder();
@@ -90,6 +113,8 @@ public struct ManifestEntry
dat.name = GetArray(ReadText(buf, ref offset));
dat.size = ReadVLQ(buf, ref offset);
dat.checksum = ReadUNum(buf, ref offset, 8);
dat.hname = CalHameString(dat.checksum, dat.size, dat.name);
dat.tname = Encoding.UTF8.GetString(dat.name);
}
}
@@ -105,6 +130,9 @@ public struct ManifestEntry
dat.priority = (uint)ReadVLQ(buf, ref offset, 4);
dat.size = (uint)ReadVLQ(buf, ref offset, 8);
dat.checksum = ReadUNum(buf, ref offset, 8);
dat.hname = CalHameString(dat.checksum, dat.size, dat.name);
dat.tname = Encoding.UTF8.GetString(dat.name);
dat.tdeps = (dat.deps == null ? null : Encoding.UTF8.GetString(dat.deps));
}
}
@@ -118,6 +146,9 @@ public struct ManifestEntry
dat.group = (uint)ReadVLQ(buf, ref offset, 4);
dat.size = (uint)ReadVLQ(buf, ref offset, 8);
dat.checksum = ReadUNum(buf, ref offset, 8);
dat.hname = CalHameString(dat.checksum, dat.size, dat.name);
dat.tname = Encoding.UTF8.GetString(dat.name);
dat.tdeps = (dat.deps == null ? null : Encoding.UTF8.GetString(dat.deps));
dat.priority = 0;
}
}
+2 -1
View File
@@ -128,5 +128,6 @@ public enum UmaFileType
uianimation,
transferevent,
teambuilding,
challengematch
challengematch,
collectevent
}