464 lines
18 KiB
Python
464 lines
18 KiB
Python
import glob
|
||
import json
|
||
import os
|
||
import shutil
|
||
from encryption import encrypt_file
|
||
from helpers import fetchKey, is_cjk
|
||
|
||
translationFixes = [
|
||
{
|
||
"key": "folder_event1",
|
||
"japaneseText": "東方Project特集",
|
||
"englishUsText": "Touhou Project",
|
||
},
|
||
{
|
||
"key": "folder_intro_event1",
|
||
"japaneseText": "東方Projectアレンジの曲をあつめたよ！",
|
||
"englishUsText": "A collection of Touhou Project songs!",
|
||
},
|
||
{
|
||
"key": "folder_event2",
|
||
"japaneseText": "アイドルマスター特集",
|
||
"englishUsText": "The Idolmaster",
|
||
},
|
||
{
|
||
"key": "folder_intro_event2",
|
||
"japaneseText": "東方Projectアレンジの曲をあつめたよ！",
|
||
"englishUsText": "A collection of songs from The Idolmaster!",
|
||
},
|
||
{
|
||
"key": "folder_event5",
|
||
"japaneseText": "スタジオジブリ特集",
|
||
"englishUsText": "Studio Ghibli",
|
||
},
|
||
{
|
||
"key": "folder_intro_event5",
|
||
"japaneseText": "東方Projectアレンジの曲をあつめたよ！",
|
||
"englishUsText": "A collection of Studio Ghibli songs!",
|
||
},
|
||
{
|
||
"key": "folder_event6",
|
||
"japaneseText": "妖怪ウォッチ特集",
|
||
"englishUsText": "Yokai Watch",
|
||
},
|
||
{
|
||
"key": "folder_intro_event6",
|
||
"japaneseText": "東方Projectアレンジの曲をあつめたよ！",
|
||
"englishUsText": "A collection of Yokai Watch songs!",
|
||
},
|
||
{
|
||
"key": "folder_event7",
|
||
"japaneseText": "UUUMクリエイター特集",
|
||
"englishUsText": "UUUM Creator Feature",
|
||
},
|
||
{
|
||
"key": "folder_intro_event7",
|
||
"japaneseText": "「#コンパス」の曲をあつめたよ！",
|
||
"englishUsText": "A collection of songs from UMMM!",
|
||
},
|
||
{
|
||
"key": "folder_event12",
|
||
"japaneseText": "#コンパス特集",
|
||
"englishUsText": "#Compass Creator Feature",
|
||
},
|
||
{
|
||
"key": "folder_intro_event12",
|
||
"japaneseText": "「#コンパス」の曲をあつめたよ！",
|
||
"englishUsText": "A collection of songs from the game #Compass!",
|
||
},
|
||
]
|
||
|
||
# region Loading json files
|
||
# Loading final song list ======================================================================
|
||
finalList = json.load(open("./temp/finalList.json", encoding="utf-8"))
|
||
|
||
|
||
# Loading wordlists ============================================================================
|
||
wordlist = json.load(open("./Data_decrypted/wordlist.json", encoding="utf-8"))
|
||
omni_wordlist_en = json.load(open("../08.18 & CHN/gamefiles/Omni/wordlist_en.json", encoding="utf-8"))
|
||
omni_wordlist_jp = json.load(open("../08.18 & CHN/gamefiles/Omni/wordlist_jp.json", encoding="utf-8"))
|
||
|
||
music_attributes = json.load(open("./Data_decrypted/music_attribute.json", encoding="utf-8"))
|
||
omni_music_attributes = json.load(open("../08.18 & CHN/gamefiles/Omni/music_attribute.json", encoding="utf-8"))
|
||
|
||
# Loading music_order ====================================================
|
||
music_orders = json.load(open("./Data_decrypted/music_order.json", encoding="utf-8"))
|
||
omni_music_orders = {"items": []}
|
||
for item in json.load(open("../08.18 & CHN/gamefiles/Omni/music_order.json", encoding="utf-8"))["items"]:
|
||
if item["genreNo"] >= 6:
|
||
item["genreNo"] -= 1
|
||
omni_music_orders["items"].append(item)
|
||
|
||
# Loading music_ai_section =============================================
|
||
music_ai_section = json.load(open("./Data_decrypted/music_ai_section.json", encoding="utf-8"))
|
||
|
||
# Loading musicinfo ====================================================
|
||
musicinfos = json.load(open("./Data_decrypted/musicinfo.json", encoding="utf-8"))
|
||
|
||
omni_musicinfos = {"items": []}
|
||
for item in json.load(open("../08.18 & CHN/gamefiles/Omni/musicinfo.json", encoding="utf-8"))["items"]:
|
||
if item["genreNo"] >= 6:
|
||
item["genreNo"] -= 1
|
||
omni_musicinfos["items"].append(item)
|
||
# endregion
|
||
|
||
|
||
# region Game files
|
||
###################
|
||
###################
|
||
#### endregion ####
|
||
|
||
|
||
# region musicinfo.json, music_usbsetting.json, music_attributes.json, music_ai_section.json.
|
||
for newentry in finalList["songs"]:
|
||
# we try to find an entry from the final list in the 39.06's musicinfo file
|
||
entry = next((item for item in musicinfos["items"] if item["id"] == newentry["id"]), None)
|
||
|
||
# if we find nothing that means the song is from omnimix
|
||
if entry is None:
|
||
# we get the musicinfo entry from the omnimix files and append it to the 39.06 file.
|
||
omni_entry = next((item for item in omni_musicinfos["items"] if item["id"] == newentry["id"]), None)
|
||
omni_entry["spikeOnEasy"] = 0
|
||
omni_entry["spikeOnNormal"] = 0
|
||
omni_entry["spikeOnHard"] = 0
|
||
omni_entry["spikeOnOni"] = 0
|
||
omni_entry["spikeOnUra"] = 0
|
||
musicinfos["items"].append(omni_entry)
|
||
|
||
|
||
# we generate a list of all unused uniqueIds bellow 1599 and bellow the highest uniqueId for a 39.06 song.
|
||
higher = 0
|
||
usedUniqueIds = []
|
||
for song in musicinfos["items"]:
|
||
uniqueId = song["uniqueId"]
|
||
# find higher id
|
||
usedUniqueIds.append(uniqueId)
|
||
if uniqueId >= higher:
|
||
higher = uniqueId
|
||
unusedList = []
|
||
overLimitList = []
|
||
for i in range(higher):
|
||
if all([i not in usedUniqueIds, i <= 1599]):
|
||
unusedList.append(i)
|
||
if all([i in usedUniqueIds, i > 1599]):
|
||
overLimitList.append(i)
|
||
|
||
|
||
# we then remap all songs above id 1599 using the list of unused uniqueIds bellow 1599.
|
||
unusedIndex = 0
|
||
remapJson = {"items": []}
|
||
for song in musicinfos["items"]:
|
||
if song["uniqueId"] > 1599:
|
||
if len(unusedList) > 0:
|
||
if unusedIndex < len(unusedList):
|
||
remapJson["items"].append(
|
||
{"id": song["id"], "uniqueIdOriginal": song["uniqueId"], "uniqueIdRemap": unusedList[unusedIndex]}
|
||
)
|
||
song["uniqueId"] = unusedList[unusedIndex]
|
||
unusedIndex += 1
|
||
music_ai_section["items"].append(
|
||
{
|
||
"id": song["id"],
|
||
"uniqueId": song["uniqueId"],
|
||
"easy": 5,
|
||
"normal": 5,
|
||
"hard": 5,
|
||
"oni": 5,
|
||
"ura": 3,
|
||
"oniLevel11": "",
|
||
"uraLevel11": "",
|
||
},
|
||
)
|
||
else:
|
||
print("Couldn't remap " + song["id"])
|
||
else:
|
||
print("Couldn't remap " + song["id"])
|
||
print("Remapped " + str(len(remapJson["items"])) + " songs")
|
||
json_object = json.dumps(remapJson["items"], indent="\t", ensure_ascii=False)
|
||
with open("./Data_exported/Data_mods/x64/datatable/dec/remap.json", "w", encoding="utf8") as outfile:
|
||
outfile.write(json_object)
|
||
|
||
# we generate the music_ai_section file
|
||
music_ai_section["items"].sort(key=lambda x: x["uniqueId"], reverse=False)
|
||
json_object = json.dumps(music_ai_section, indent="\t", ensure_ascii=False)
|
||
with open("./Data_exported/Data_mods/x64/datatable/dec/music_ai_section.json", "w", encoding="utf8") as outfile:
|
||
outfile.write(json_object)
|
||
|
||
|
||
# we generate the music_usbsetting file
|
||
usbsettingjson = {"items": []}
|
||
musicinfos["items"].sort(key=lambda x: x["uniqueId"], reverse=False)
|
||
for song in musicinfos["items"]:
|
||
usbsetting = {"id": song["id"], "uniqueId": int(song["uniqueId"]), "usbVer": ""}
|
||
usbsettingjson["items"].append(usbsetting)
|
||
json_object = json.dumps(usbsettingjson, indent="\t", ensure_ascii=False)
|
||
with open("./Data_exported/Data_mods/x64/datatable/dec/music_usbsetting.json", "w", encoding="utf8") as outfile:
|
||
outfile.write(json_object)
|
||
|
||
|
||
# we generate the music_attribute file
|
||
for song in musicinfos["items"]:
|
||
entry = next((item for item in music_attributes["items"] if item["id"] == song["id"]), None)
|
||
# if we find nothing that means the song is from omnimix
|
||
if entry is None:
|
||
# we get the musicinfo entry from the omnimix files and append it to the 39.06 file.
|
||
omni_entry = next((item for item in omni_music_attributes["items"] if item["id"] == song["id"]), None)
|
||
omni_entry["uniqueId"] = song["uniqueId"]
|
||
omni_entry["ensoPartsID1"] = 0
|
||
omni_entry["ensoPartsID2"] = 0
|
||
del omni_entry["canPlayUra"]
|
||
music_attributes["items"].append(omni_entry)
|
||
music_attributes["items"].sort(key=lambda x: x["uniqueId"], reverse=False)
|
||
json_object = json.dumps(music_attributes, indent="\t", ensure_ascii=False)
|
||
with open("./Data_exported/Data_mods/x64/datatable/dec/music_attribute.json", "w", encoding="utf8") as outfile:
|
||
outfile.write(json_object)
|
||
|
||
musicinfos["items"].sort(key=lambda x: x["uniqueId"], reverse=False)
|
||
json_object = json.dumps(musicinfos, indent="\t", ensure_ascii=False)
|
||
with open("./Data_exported/Data_mods/x64/datatable/dec/musicinfo.json", "w", encoding="utf8") as outfile:
|
||
outfile.write(json_object)
|
||
|
||
print("Wrote musicinfo, music_attribute and music_usbsetting.\n")
|
||
# # endregion
|
||
|
||
# region wordlist.json
|
||
for song in finalList["songs"]:
|
||
songKey = "song_" + song["id"]
|
||
songSubKey = "song_sub_" + song["id"]
|
||
songDetailKey = "song_sub_" + song["id"]
|
||
|
||
# song entry
|
||
entry = next((item for item in wordlist["items"] if item["key"] == songKey), None)
|
||
if entry is not None:
|
||
if entry["japaneseText"] != "":
|
||
next
|
||
else:
|
||
print(songKey, "is already in the wordlist but has an empty string.")
|
||
entry["japaneseText"] = song["nameJp"]
|
||
entry["englishUsText"] = song["nameUs"]
|
||
entry["englishUsFontType"] = 1 if not is_cjk(song["nameUs"]) else 0
|
||
else:
|
||
print(songKey, "has been added to the wordlist.")
|
||
wordlist["items"].append(
|
||
{
|
||
"key": songKey,
|
||
"japaneseText": song["nameJp"],
|
||
"japaneseFontType": 0,
|
||
"englishUsText": song["nameUs"],
|
||
"englishUsFontType": 1 if not is_cjk(song["nameUs"]) else 0,
|
||
},
|
||
)
|
||
|
||
# song sub entry
|
||
entry = next((item for item in wordlist["items"] if item["key"] == songSubKey), None)
|
||
if entry is not None:
|
||
if entry["japaneseText"] != "":
|
||
next
|
||
else:
|
||
subentry = next(
|
||
(item for item in omni_wordlist_en["items"] if item["key"] == songSubKey),
|
||
{"japaneseText": ""},
|
||
)["japaneseText"]
|
||
if subentry != "":
|
||
print(songKey, "sub is already in the wordlist but has an empty string.")
|
||
entry["japaneseText"] = subentry
|
||
entry["englishUsText"] = subentry
|
||
entry["englishUsFontType"] = 1 if not is_cjk(subentry) else 0
|
||
else:
|
||
subentry = next(
|
||
(item for item in omni_wordlist_en["items"] if item["key"] == songSubKey),
|
||
{"japaneseText": ""},
|
||
)["japaneseText"]
|
||
|
||
wordlist["items"].append(
|
||
{
|
||
"key": songSubKey,
|
||
"japaneseText": subentry,
|
||
"japaneseFontType": 0,
|
||
"englishUsText": subentry,
|
||
"englishUsFontType": 1 if not is_cjk(subentry) else 0,
|
||
},
|
||
)
|
||
if subentry != "":
|
||
print(songSubKey, "has been added to the wordlist.")
|
||
|
||
# song detail entry
|
||
entry = next((item for item in wordlist["items"] if item["key"] == songDetailKey), None)
|
||
if entry is not None:
|
||
if entry["japaneseText"] != "":
|
||
next
|
||
else:
|
||
detailentry = next(
|
||
(item for item in omni_wordlist_en["items"] if item["key"] == songDetailKey),
|
||
{"japaneseText": ""},
|
||
)["japaneseText"]
|
||
if detailentry != "":
|
||
print(songKey, "detail is already in the wordlist but has an empty string.")
|
||
entry["japaneseText"] = detailentry
|
||
entry["englishUsText"] = detailentry
|
||
entry["englishUsFontType"] = 1 if not is_cjk(detailentry) else 0
|
||
else:
|
||
detailentry = next(
|
||
(item for item in omni_wordlist_en["items"] if item["key"] == songDetailKey),
|
||
{"japaneseText": ""},
|
||
)["japaneseText"]
|
||
if detailentry != "":
|
||
print(songDetailKey, "has been added to the wordlist.")
|
||
wordlist["items"].append(
|
||
{
|
||
"key": songDetailKey,
|
||
"japaneseText": detailentry,
|
||
"japaneseFontType": 0,
|
||
"englishUsText": detailentry,
|
||
"englishUsFontType": 1 if not is_cjk(detailentry) else 0,
|
||
},
|
||
)
|
||
print("Processed wordlist.\n")
|
||
# endregion
|
||
|
||
|
||
# region music_order.json
|
||
|
||
# closedisptype in music_order
|
||
# 1 to show subtitle
|
||
# 0 to show title
|
||
for song in music_attributes["items"]:
|
||
# we try to find an entry from the final list in the 39.06's music_order file
|
||
entry = next((item for item in music_orders["items"] if item["id"] == song["id"]), None)
|
||
name = next((item for item in wordlist["items"] if item["key"] == "song_" + song["id"]), {"englishUsText": ""})
|
||
|
||
if name["englishUsText"] == "" and song["id"] != "tmap4":
|
||
name["englishUsText"] = name["japaneseText"]
|
||
name["englishUsFontType"] = 1 if not is_cjk(name["japaneseText"]) else 0
|
||
print("Missing title for", name["key"])
|
||
|
||
# if we find nothing that means the song is from omnimix
|
||
if entry is None:
|
||
if song["id"] != "tmap4":
|
||
for omniEntry in omni_music_orders["items"]:
|
||
if omniEntry["id"] == song["id"]:
|
||
omniEntry["uniqueId"] = song["uniqueId"]
|
||
omniEntry["englishUsText"] = name["englishUsText"]
|
||
music_orders["items"].append(omniEntry)
|
||
continue
|
||
else:
|
||
for entry in music_orders["items"]:
|
||
if entry["id"] == song["id"]:
|
||
entry["englishUsText"] = name["englishUsText"]
|
||
|
||
# Writing music_order ===============================================================================
|
||
# ordering music_order by genre and english name
|
||
music_orders["items"].sort(key=lambda x: (x["genreNo"], x["englishUsText"]))
|
||
# removing the names from the dict
|
||
# for items in music_orders["items"]:
|
||
# if "englishUsText" in items:
|
||
# del items["englishUsText"]
|
||
# writing the music order
|
||
json_object = json.dumps(music_orders, indent="\t", ensure_ascii=False)
|
||
with open("./Data_exported/Data_mods/x64/datatable/dec/music_order.json", "w", encoding="utf8") as outfile:
|
||
outfile.write(json_object)
|
||
print("Wrote music_order.\n")
|
||
|
||
|
||
# wordlist["items"].sort(key=lambda x: x["key"], reverse=False)
|
||
# removing unused languages from the dict
|
||
for items in wordlist["items"]:
|
||
if "koreanText" in items:
|
||
del items["koreanText"]
|
||
del items["koreanFontType"]
|
||
# # if "chineseTText" in items:
|
||
# # del items["chineseTText"]
|
||
# # del items["chineseTFontType"]
|
||
|
||
for entry in translationFixes:
|
||
key = fetchKey(key=entry["key"], wordlist=wordlist)
|
||
key["englishUsText"] = entry["englishUsText"]
|
||
key["englishUsFontType"] = 1
|
||
|
||
print(key["japaneseText"], "->", key["englishUsText"])
|
||
|
||
# exporting the wordlist.
|
||
json_object = json.dumps(wordlist, ensure_ascii=False, indent="\t")
|
||
with open("./Data_exported/Data_mods/x64/datatable/dec/wordlist.json", "w", encoding="utf8") as outfile:
|
||
outfile.write(json_object)
|
||
# Fixing missing translations
|
||
print("Wrote wordlist.\n")
|
||
# endregion
|
||
|
||
|
||
# region Encrypting databases
|
||
files = glob.glob("./Data_exported/Data_mods/x64/datatable/dec/*")
|
||
for f in files:
|
||
outfile = os.path.splitext(os.path.basename(f))[0] + ".bin"
|
||
outdir = os.path.join("./Data_exported/Data_mods/x64/datatable/", outfile)
|
||
if outfile != "remap.bin":
|
||
print("Encrypting " + f + " to " + outdir)
|
||
file = encrypt_file(input_file=f)
|
||
with open(outdir, "wb") as outfile:
|
||
outfile.write(file)
|
||
print("Encrypted Datatables.\n")
|
||
# endregion
|
||
|
||
|
||
# region Writing server files
|
||
#############################
|
||
ServerFolderSongsPerType = 20
|
||
######## endregion ##########
|
||
|
||
|
||
# region event_folder_data.json
|
||
playcounts = json.load(open(file="./temp/listPlays.json", encoding="utf-8"))
|
||
eventfolders = json.load(open(file="./Data_decrypted/Server/event_folder_data.json", encoding="utf-8"))
|
||
musicinfo = json.load(open(file="./Data_exported/Data_mods/x64/datatable/dec/musicinfo.json", encoding="utf-8"))
|
||
|
||
# The recommended song folder is accessed with the following key:
|
||
# eventfolders[2]["songNo"]
|
||
|
||
MostPlayedList = []
|
||
for song in playcounts["Omnimix"]:
|
||
song["Omni"] = True
|
||
if song not in MostPlayedList:
|
||
MostPlayedList.append(song)
|
||
if len(MostPlayedList) > ServerFolderSongsPerType:
|
||
break
|
||
for song in playcounts["Regular"]:
|
||
song["Omni"] = False
|
||
if song not in MostPlayedList:
|
||
MostPlayedList.append(song)
|
||
if len(MostPlayedList) > ServerFolderSongsPerType * 2 - 1:
|
||
break
|
||
MostPlayedList = sorted(MostPlayedList, key=lambda item: item["plays"], reverse=True)
|
||
|
||
|
||
MostPlayedArray = []
|
||
for song in MostPlayedList:
|
||
uniqueId = next((item for item in musicinfo["items"] if item["id"] == song["id"]), None)
|
||
if not uniqueId:
|
||
print(song["id"])
|
||
else:
|
||
print(str(song["plays"]).zfill(3), "=>", "O" if song["Omni"] else "R", "=>", song["id"], "=>", song["nameUs"])
|
||
MostPlayedArray.append(uniqueId["uniqueId"])
|
||
|
||
print("Exported", len(MostPlayedArray), "songs")
|
||
|
||
eventfolders[2]["songNo"] = MostPlayedArray
|
||
eventfoldersDump = json.dumps(eventfolders, indent=4, ensure_ascii=False)
|
||
with open("./Data_exported/Server/wwwroot/data/event_folder_data.json", "w", encoding="utf8") as outfile:
|
||
outfile.write(eventfoldersDump)
|
||
print("Wrote event_folder_data.\n")
|
||
# endregion
|
||
|
||
|
||
# region shop_folder_data.json
|
||
with open("./Data_exported/Server/wwwroot/data/shop_folder_data.json", "w", encoding="utf8") as outfile:
|
||
outfile.write(json.dumps([], indent=4, ensure_ascii=False))
|
||
print("Wrote shop_folder_data.\n")
|
||
# endregion
|
||
|
||
|
||
# region movie_data.json
|
||
with open("./Data_exported/Server/wwwroot/data/movie_data.json", "w", encoding="utf8") as outfile:
|
||
outfile.write(json.dumps([{"movie_id": 20, "enable_days": 999}], indent=4, ensure_ascii=False))
|
||
print("Wrote movie_data.\n")
|
||
# endregion
|