import glob import json import os import shutil from encryption import encrypt_file from helpers import fetchKey, is_cjk translationFixes = [ { "key": "folder_event1", "japaneseText": "東方Project特集", "englishUsText": "Touhou Project", }, { "key": "folder_intro_event1", "japaneseText": "東方Projectアレンジの曲をあつめたよ!", "englishUsText": "A collection of Touhou Project songs!", }, { "key": "folder_event2", "japaneseText": "アイドルマスター特集", "englishUsText": "The Idolmaster", }, { "key": "folder_intro_event2", "japaneseText": "東方Projectアレンジの曲をあつめたよ!", "englishUsText": "A collection of songs from The Idolmaster!", }, { "key": "folder_event5", "japaneseText": "スタジオジブリ特集", "englishUsText": "Studio Ghibli", }, { "key": "folder_intro_event5", "japaneseText": "東方Projectアレンジの曲をあつめたよ!", "englishUsText": "A collection of Studio Ghibli songs!", }, { "key": "folder_event6", "japaneseText": "妖怪ウォッチ特集", "englishUsText": "Yokai Watch", }, { "key": "folder_intro_event6", "japaneseText": "東方Projectアレンジの曲をあつめたよ!", "englishUsText": "A collection of Yokai Watch songs!", }, { "key": "folder_event7", "japaneseText": "UUUMクリエイター特集", "englishUsText": "UUUM Creator Feature", }, { "key": "folder_intro_event7", "japaneseText": "「#コンパス」の曲をあつめたよ!", "englishUsText": "A collection of songs from UMMM!", }, { "key": "folder_event12", "japaneseText": "#コンパス特集", "englishUsText": "#Compass Creator Feature", }, { "key": "folder_intro_event12", "japaneseText": "「#コンパス」の曲をあつめたよ!", "englishUsText": "A collection of songs from the game #Compass!", }, ] # region Loading json files # Loading final song list ====================================================================== finalList = json.load(open("./temp/finalList.json", encoding="utf-8")) # Loading wordlists ============================================================================ wordlist = json.load(open("./Data_decrypted/wordlist.json", encoding="utf-8")) omni_wordlist_en = json.load(open("../08.18 & CHN/gamefiles/Omni/wordlist_en.json", encoding="utf-8")) omni_wordlist_jp = json.load(open("../08.18 & CHN/gamefiles/Omni/wordlist_jp.json", encoding="utf-8")) music_attributes = json.load(open("./Data_decrypted/music_attribute.json", encoding="utf-8")) omni_music_attributes = json.load(open("../08.18 & CHN/gamefiles/Omni/music_attribute.json", encoding="utf-8")) # Loading music_order ==================================================== music_orders = json.load(open("./Data_decrypted/music_order.json", encoding="utf-8")) omni_music_orders = {"items": []} for item in json.load(open("../08.18 & CHN/gamefiles/Omni/music_order.json", encoding="utf-8"))["items"]: if item["genreNo"] >= 6: item["genreNo"] -= 1 omni_music_orders["items"].append(item) # Loading music_ai_section ============================================= music_ai_section = json.load(open("./Data_decrypted/music_ai_section.json", encoding="utf-8")) # Loading musicinfo ==================================================== musicinfos = json.load(open("./Data_decrypted/musicinfo.json", encoding="utf-8")) omni_musicinfos = {"items": []} for item in json.load(open("../08.18 & CHN/gamefiles/Omni/musicinfo.json", encoding="utf-8"))["items"]: if item["genreNo"] >= 6: item["genreNo"] -= 1 omni_musicinfos["items"].append(item) # endregion # region Game files ################### ################### #### endregion #### # region musicinfo.json, music_usbsetting.json, music_attributes.json, music_ai_section.json. for newentry in finalList["songs"]: # we try to find an entry from the final list in the 39.06's musicinfo file entry = next((item for item in musicinfos["items"] if item["id"] == newentry["id"]), None) # if we find nothing that means the song is from omnimix if entry is None: # we get the musicinfo entry from the omnimix files and append it to the 39.06 file. omni_entry = next((item for item in omni_musicinfos["items"] if item["id"] == newentry["id"]), None) omni_entry["spikeOnEasy"] = 0 omni_entry["spikeOnNormal"] = 0 omni_entry["spikeOnHard"] = 0 omni_entry["spikeOnOni"] = 0 omni_entry["spikeOnUra"] = 0 musicinfos["items"].append(omni_entry) # we generate a list of all unused uniqueIds bellow 1599 and bellow the highest uniqueId for a 39.06 song. higher = 0 usedUniqueIds = [] for song in musicinfos["items"]: uniqueId = song["uniqueId"] # find higher id usedUniqueIds.append(uniqueId) if uniqueId >= higher: higher = uniqueId unusedList = [] overLimitList = [] for i in range(higher): if all([i not in usedUniqueIds, i <= 1599]): unusedList.append(i) if all([i in usedUniqueIds, i > 1599]): overLimitList.append(i) # we then remap all songs above id 1599 using the list of unused uniqueIds bellow 1599. unusedIndex = 0 remapJson = {"items": []} for song in musicinfos["items"]: if song["uniqueId"] > 1599: if len(unusedList) > 0: if unusedIndex < len(unusedList): remapJson["items"].append( {"id": song["id"], "uniqueIdOriginal": song["uniqueId"], "uniqueIdRemap": unusedList[unusedIndex]} ) song["uniqueId"] = unusedList[unusedIndex] unusedIndex += 1 music_ai_section["items"].append( { "id": song["id"], "uniqueId": song["uniqueId"], "easy": 5, "normal": 5, "hard": 5, "oni": 5, "ura": 3, "oniLevel11": "", "uraLevel11": "", }, ) else: print("Couldn't remap " + song["id"]) else: print("Couldn't remap " + song["id"]) print("Remapped " + str(len(remapJson["items"])) + " songs") json_object = json.dumps(remapJson["items"], indent="\t", ensure_ascii=False) with open("./Data_exported/Data_mods/x64/datatable/dec/remap.json", "w", encoding="utf8") as outfile: outfile.write(json_object) # we generate the music_ai_section file music_ai_section["items"].sort(key=lambda x: x["uniqueId"], reverse=False) json_object = json.dumps(music_ai_section, indent="\t", ensure_ascii=False) with open("./Data_exported/Data_mods/x64/datatable/dec/music_ai_section.json", "w", encoding="utf8") as outfile: outfile.write(json_object) # we generate the music_usbsetting file usbsettingjson = {"items": []} musicinfos["items"].sort(key=lambda x: x["uniqueId"], reverse=False) for song in musicinfos["items"]: usbsetting = {"id": song["id"], "uniqueId": int(song["uniqueId"]), "usbVer": ""} usbsettingjson["items"].append(usbsetting) json_object = json.dumps(usbsettingjson, indent="\t", ensure_ascii=False) with open("./Data_exported/Data_mods/x64/datatable/dec/music_usbsetting.json", "w", encoding="utf8") as outfile: outfile.write(json_object) # we generate the music_attribute file for song in musicinfos["items"]: entry = next((item for item in music_attributes["items"] if item["id"] == song["id"]), None) # if we find nothing that means the song is from omnimix if entry is None: # we get the musicinfo entry from the omnimix files and append it to the 39.06 file. omni_entry = next((item for item in omni_music_attributes["items"] if item["id"] == song["id"]), None) omni_entry["uniqueId"] = song["uniqueId"] omni_entry["ensoPartsID1"] = 0 omni_entry["ensoPartsID2"] = 0 del omni_entry["canPlayUra"] music_attributes["items"].append(omni_entry) music_attributes["items"].sort(key=lambda x: x["uniqueId"], reverse=False) json_object = json.dumps(music_attributes, indent="\t", ensure_ascii=False) with open("./Data_exported/Data_mods/x64/datatable/dec/music_attribute.json", "w", encoding="utf8") as outfile: outfile.write(json_object) musicinfos["items"].sort(key=lambda x: x["uniqueId"], reverse=False) json_object = json.dumps(musicinfos, indent="\t", ensure_ascii=False) with open("./Data_exported/Data_mods/x64/datatable/dec/musicinfo.json", "w", encoding="utf8") as outfile: outfile.write(json_object) print("Wrote musicinfo, music_attribute and music_usbsetting.\n") # # endregion # region wordlist.json for song in finalList["songs"]: songKey = "song_" + song["id"] songSubKey = "song_sub_" + song["id"] songDetailKey = "song_sub_" + song["id"] # song entry entry = next((item for item in wordlist["items"] if item["key"] == songKey), None) if entry is not None: if entry["japaneseText"] != "": next else: print(songKey, "is already in the wordlist but has an empty string.") entry["japaneseText"] = song["nameJp"] entry["englishUsText"] = song["nameUs"] entry["englishUsFontType"] = 1 if not is_cjk(song["nameUs"]) else 0 else: print(songKey, "has been added to the wordlist.") wordlist["items"].append( { "key": songKey, "japaneseText": song["nameJp"], "japaneseFontType": 0, "englishUsText": song["nameUs"], "englishUsFontType": 1 if not is_cjk(song["nameUs"]) else 0, }, ) # song sub entry entry = next((item for item in wordlist["items"] if item["key"] == songSubKey), None) if entry is not None: if entry["japaneseText"] != "": next else: subentry = next( (item for item in omni_wordlist_en["items"] if item["key"] == songSubKey), {"japaneseText": ""}, )["japaneseText"] if subentry != "": print(songKey, "sub is already in the wordlist but has an empty string.") entry["japaneseText"] = subentry entry["englishUsText"] = subentry entry["englishUsFontType"] = 1 if not is_cjk(subentry) else 0 else: subentry = next( (item for item in omni_wordlist_en["items"] if item["key"] == songSubKey), {"japaneseText": ""}, )["japaneseText"] wordlist["items"].append( { "key": songSubKey, "japaneseText": subentry, "japaneseFontType": 0, "englishUsText": subentry, "englishUsFontType": 1 if not is_cjk(subentry) else 0, }, ) if subentry != "": print(songSubKey, "has been added to the wordlist.") # song detail entry entry = next((item for item in wordlist["items"] if item["key"] == songDetailKey), None) if entry is not None: if entry["japaneseText"] != "": next else: detailentry = next( (item for item in omni_wordlist_en["items"] if item["key"] == songDetailKey), {"japaneseText": ""}, )["japaneseText"] if detailentry != "": print(songKey, "detail is already in the wordlist but has an empty string.") entry["japaneseText"] = detailentry entry["englishUsText"] = detailentry entry["englishUsFontType"] = 1 if not is_cjk(detailentry) else 0 else: detailentry = next( (item for item in omni_wordlist_en["items"] if item["key"] == songDetailKey), {"japaneseText": ""}, )["japaneseText"] if detailentry != "": print(songDetailKey, "has been added to the wordlist.") wordlist["items"].append( { "key": songDetailKey, "japaneseText": detailentry, "japaneseFontType": 0, "englishUsText": detailentry, "englishUsFontType": 1 if not is_cjk(detailentry) else 0, }, ) print("Processed wordlist.\n") # endregion # region music_order.json # closedisptype in music_order # 1 to show subtitle # 0 to show title for song in music_attributes["items"]: # we try to find an entry from the final list in the 39.06's music_order file entry = next((item for item in music_orders["items"] if item["id"] == song["id"]), None) name = next((item for item in wordlist["items"] if item["key"] == "song_" + song["id"]), {"englishUsText": ""}) if name["englishUsText"] == "" and song["id"] != "tmap4": name["englishUsText"] = name["japaneseText"] name["englishUsFontType"] = 1 if not is_cjk(name["japaneseText"]) else 0 print("Missing title for", name["key"]) # if we find nothing that means the song is from omnimix if entry is None: if song["id"] != "tmap4": for omniEntry in omni_music_orders["items"]: if omniEntry["id"] == song["id"]: omniEntry["uniqueId"] = song["uniqueId"] omniEntry["englishUsText"] = name["englishUsText"] music_orders["items"].append(omniEntry) continue else: for entry in music_orders["items"]: if entry["id"] == song["id"]: entry["englishUsText"] = name["englishUsText"] # Writing music_order =============================================================================== # ordering music_order by genre and english name music_orders["items"].sort(key=lambda x: (x["genreNo"], x["englishUsText"])) # removing the names from the dict # for items in music_orders["items"]: # if "englishUsText" in items: # del items["englishUsText"] # writing the music order json_object = json.dumps(music_orders, indent="\t", ensure_ascii=False) with open("./Data_exported/Data_mods/x64/datatable/dec/music_order.json", "w", encoding="utf8") as outfile: outfile.write(json_object) print("Wrote music_order.\n") # wordlist["items"].sort(key=lambda x: x["key"], reverse=False) # removing unused languages from the dict for items in wordlist["items"]: if "koreanText" in items: del items["koreanText"] del items["koreanFontType"] # # if "chineseTText" in items: # # del items["chineseTText"] # # del items["chineseTFontType"] for entry in translationFixes: key = fetchKey(key=entry["key"], wordlist=wordlist) key["englishUsText"] = entry["englishUsText"] key["englishUsFontType"] = 1 print(key["japaneseText"], "->", key["englishUsText"]) # exporting the wordlist. json_object = json.dumps(wordlist, ensure_ascii=False, indent="\t") with open("./Data_exported/Data_mods/x64/datatable/dec/wordlist.json", "w", encoding="utf8") as outfile: outfile.write(json_object) # Fixing missing translations print("Wrote wordlist.\n") # endregion # region Encrypting databases files = glob.glob("./Data_exported/Data_mods/x64/datatable/dec/*") for f in files: outfile = os.path.splitext(os.path.basename(f))[0] + ".bin" outdir = os.path.join("./Data_exported/Data_mods/x64/datatable/", outfile) if outfile != "remap.bin": print("Encrypting " + f + " to " + outdir) file = encrypt_file(input_file=f) with open(outdir, "wb") as outfile: outfile.write(file) print("Encrypted Datatables.\n") # endregion # region Writing server files ############################# ServerFolderSongsPerType = 20 ######## endregion ########## # region event_folder_data.json playcounts = json.load(open(file="./temp/listPlays.json", encoding="utf-8")) eventfolders = json.load(open(file="./Data_decrypted/Server/event_folder_data.json", encoding="utf-8")) musicinfo = json.load(open(file="./Data_exported/Data_mods/x64/datatable/dec/musicinfo.json", encoding="utf-8")) # The recommended song folder is accessed with the following key: # eventfolders[2]["songNo"] MostPlayedList = [] for song in playcounts["Omnimix"]: song["Omni"] = True if song not in MostPlayedList: MostPlayedList.append(song) if len(MostPlayedList) > ServerFolderSongsPerType: break for song in playcounts["Regular"]: song["Omni"] = False if song not in MostPlayedList: MostPlayedList.append(song) if len(MostPlayedList) > ServerFolderSongsPerType * 2 - 1: break MostPlayedList = sorted(MostPlayedList, key=lambda item: item["plays"], reverse=True) MostPlayedArray = [] for song in MostPlayedList: uniqueId = next((item for item in musicinfo["items"] if item["id"] == song["id"]), None) if not uniqueId: print(song["id"]) else: print(str(song["plays"]).zfill(3), "=>", "O" if song["Omni"] else "R", "=>", song["id"], "=>", song["nameUs"]) MostPlayedArray.append(uniqueId["uniqueId"]) print("Exported", len(MostPlayedArray), "songs") eventfolders[2]["songNo"] = MostPlayedArray eventfoldersDump = json.dumps(eventfolders, indent=4, ensure_ascii=False) with open("./Data_exported/Server/wwwroot/data/event_folder_data.json", "w", encoding="utf8") as outfile: outfile.write(eventfoldersDump) print("Wrote event_folder_data.\n") # endregion # region shop_folder_data.json with open("./Data_exported/Server/wwwroot/data/shop_folder_data.json", "w", encoding="utf8") as outfile: outfile.write(json.dumps([], indent=4, ensure_ascii=False)) print("Wrote shop_folder_data.\n") # endregion # region movie_data.json with open("./Data_exported/Server/wwwroot/data/movie_data.json", "w", encoding="utf8") as outfile: outfile.write(json.dumps([{"movie_id": 20, "enable_days": 999}], indent=4, ensure_ascii=False)) print("Wrote movie_data.\n") # endregion