Taiko/tooling/deleteMissingSongs.py

# -*- coding: utf-8 -*-
import os
import sqlite3
import numpy as np
import json
from dotenv import load_dotenv

load_dotenv()

musicinfo_list = json.load(open(file="./Data_exported/Data_mods/x64/datatable/dec/musicinfo.json", encoding="utf-8"))[
    "items"
]

unique_id_list = [entry["uniqueId"] for entry in musicinfo_list]
unique_id_list = np.unique(unique_id_list)
unique_id_list = sorted(unique_id_list)

# Connect to the database
conn = sqlite3.connect("./Data_exported/Server/wwwroot/taiko.db3")
c = conn.cursor()

# Get the AiScoreData table
c.execute("SELECT * FROM AiScoreData")
ai_score_data_list = c.fetchall()
# Column 1 is the SongId
new_ai_score_data_list = [row for row in ai_score_data_list if int(row[1]) in unique_id_list]


# Write back to the database
c.execute("DELETE FROM AiScoreData")
query = "INSERT INTO AiScoreData VALUES (" + ",".join(["?"] * len(new_ai_score_data_list[0])) + ")"
c.executemany(query, new_ai_score_data_list)

# Get the AiSectionScoreData table
c.execute("SELECT * FROM AiSectionScoreData")
ai_section_score_data_list = c.fetchall()
# Column 1 is the SongId
new_ai_section_score_data_list = [row for row in ai_section_score_data_list if int(row[1]) in unique_id_list]

# Write back to the database
c.execute("DELETE FROM AiSectionScoreData")
query = "INSERT INTO AiSectionScoreData VALUES (" + ",".join(["?"] * len(new_ai_section_score_data_list[0])) + ")"
c.executemany(query, new_ai_section_score_data_list)

# Get the SongBestData table
c.execute("SELECT * FROM SongBestData")
song_best_data_list = c.fetchall()
# Column 1 is the SongId
new_song_best_data_list = [row for row in song_best_data_list if int(row[1]) in unique_id_list]

# Write back to the database
c.execute("DELETE FROM SongBestData")
query = "INSERT INTO SongBestData VALUES (" + ",".join(["?"] * len(new_song_best_data_list[0])) + ")"
c.executemany(query, new_song_best_data_list)

# Get the SongPlayData table
c.execute("SELECT * FROM SongPlayData")
song_play_data_list = c.fetchall()
# Column -2 is the SongId
new_song_play_data_list = [row for row in song_play_data_list if int(row[-2]) in unique_id_list]
removed_song_play_data_list = [row for row in song_play_data_list if int(row[-2]) not in unique_id_list]

# print(removed_song_play_data_list)

removed_id_list = []
for entry in removed_song_play_data_list:
    if entry[-2] not in removed_id_list:
        removed_id_list.append(entry[-2])
removed_id_list = sorted(removed_id_list)

print("removed", len(removed_id_list), "ids", removed_id_list)

# Write back to the database
c.execute("DELETE FROM SongPlayData")
query = "INSERT INTO SongPlayData VALUES (" + ",".join(["?"] * len(new_song_play_data_list[0])) + ")"
c.executemany(query, new_song_play_data_list)

# Get the UserData table
c.execute("SELECT * FROM UserData")
user_data_list = c.fetchall()
new_user_data_list = []
for row in user_data_list:
    new_row = list(row)
    # Column 12 is the FavoriteSongsArray
    favorite_songs_array = json.loads(row[12])
    new_favorite_songs_array = [song_id for song_id in favorite_songs_array if int(song_id) in unique_id_list]
    new_row[12] = json.dumps(new_favorite_songs_array)

    # Column 28 is the UnlockedSongIdList
    try:
        unlocked_song_id_list = json.loads(row[28])
    except:
        unlocked_song_id_list = []

    new_unlocked_song_id_list = [song_id for song_id in unlocked_song_id_list if int(song_id) in unique_id_list]
    new_row[28] = json.dumps(new_unlocked_song_id_list)
    new_user_data_list.append(new_row)

# Write back to the database
c.execute("DELETE FROM UserData")
query = "INSERT INTO UserData VALUES (" + ",".join(["?"] * len(new_user_data_list[0])) + ")"
c.executemany(query, new_user_data_list)

conn.commit()
conn.close()