remove max_song_id, add partial_entries and exclude_omni

dist/popnhax/popnhax.xml

@@ -131,7 +131,7 @@
                 THREAD_PRIORITY_TIME_CRITICAL  15                             -->
     <enhanced_polling_priority __type="s8">1</enhanced_polling_priority>
 
-  <!-- Song db patches -->
+  <!-- Song db patches (requires patch_db) -->
     <!-- Auto select patch file from data_mods folder based on music limit, or datecode otherwise (will detect datecode from ea3-config or force_datecode option) -->
     <patch_xml_auto __type="bool">1</patch_xml_auto>
     <!-- Manually set XML file containing patches (requires patch_xml_auto to be disabled) -->
@@ -143,11 +143,13 @@
     <!-- Do not perform music limit checks for patch_xml_auto (not recommended) -->
     <ignore_music_limit __type="bool">0</ignore_music_limit>
 
-  <!-- Custom category options -->
-    <!-- minimum songid for a song to be seen as "custom" -->
-    <custom_categ_min_songid __type="u16">4000</custom_categ_min_songid>
-    <!-- maximum songid for a song to be seen as "custom" (0 = no limit, not recommended since "random" category will use fake song ids in the 65400+ range) -->
-    <custom_categ_max_songid __type="u16">65400</custom_categ_max_songid>
+  <!-- Custom category options (requires patch_db) -->
+    <!-- Also exclude omnimix (song id < 3000) tracks from version/level (requires custom_exclude_from_level or custom_exclude_from_version) -->
+    <exclude_omni __type="bool">0</exclude_omni>
+    <!-- New chart added to existing song moves the song to custom folder -->
+    <partial_entries __type="bool">0</partial_entries>
+    <!-- Minimum songid for a song to be seen as "custom" (e.g. use 4000 for real customs only) -->
+    <custom_categ_min_songid __type="u16">0</custom_categ_min_songid>
     <!-- Category title for customs -->
     <custom_category_title __type="str">Customs</custom_category_title>
     <!-- Format used for category title (in BM2DXFontScript format, refer to BM2DXFontScript.md in popnhax_tools repo)  -->

popnhax/config.h

@@ -17,7 +17,6 @@ struct popnhax_config {
     bool score_challenge;
     uint8_t custom_categ;
     uint16_t custom_categ_min_songid;
-    uint16_t custom_categ_max_songid;
     bool custom_exclude_from_version;
     bool custom_exclude_from_level;
     bool force_hd_timing;
@@ -64,6 +63,8 @@ struct popnhax_config {
     char custom_category_format[64];
     char custom_track_title_format[64];
     char custom_track_title_format2[64];
+    bool exclude_omni;
+    bool partial_entries;
 };
 
 #endif

popnhax/custom_categs.cc

@@ -19,14 +19,10 @@
 #include "minhook/hde32.h"
 #include "minhook/include/MinHook.h"
 
+#include "custom_categs.h"
+
 #define F_OK 0
 
-uint8_t g_game_version;
-uint32_t g_playerdata_ptr_addr; //pointer to the playerdata memory zone (offset 0x08 is popn friend ID as ascii (12 char long), offset 0x1A5 is "is logged in" flag)
-char *g_current_friendid;
-uint32_t g_current_songid;
-
-void (*add_song_in_list)();
 //game code takes array start address from offset 0xC and the address after the list end from offset 0x10
 typedef struct songlist_s {
     uint32_t dummy[3];
@@ -34,9 +30,17 @@ typedef struct songlist_s {
     uint32_t array_end;
 } songlist_t;
 
+uint8_t g_game_version;
+uint32_t g_playerdata_ptr_addr; //pointer to the playerdata memory zone (offset 0x08 is popn friend ID as ascii (12 char long), offset 0x1A5 is "is logged in" flag)
+char *g_current_friendid;
+uint32_t g_current_songid;
+
+bst_t *g_customs_bst = NULL;
+bool g_exclude_omni = false;
+
+void (*add_song_in_list)();
+
 bool g_subcategmode = false;
-uint32_t g_min_id = 4000;
-uint32_t g_max_id = 0;
 
 const char *g_categicon;
 const char *g_categformat;
@@ -57,7 +61,12 @@ uint32_t favorites_struct_addr = (uint32_t)&favorites_struct;
 
 bool is_a_custom(uint32_t songid)
 {
-	return (songid >= g_min_id && (g_max_id==0 || g_max_id >= songid));
+    return (bst_search(g_customs_bst, songid) != NULL);
+}
+
+bool is_excluded_from_level(uint32_t songid)
+{
+    return ( is_a_custom(songid) && ( g_exclude_omni || songid >= 3000 ) );
 }
 
 void add_song_to_favorites()
@@ -238,12 +247,6 @@ uint32_t songlist_count = 0;
 songlist_t songlist_struct;
 uint32_t songlist_struct_addr = (uint32_t)&songlist_struct;
 
-typedef struct {
-    char *name;
-    uint32_t size;
-    uint32_t *songlist;
-} subcategory_s;
-
 subcategory_s* subcategories;
 uint32_t g_subcateg_count = 0;
 //subcategories[0] is the "ALL SONGS" virtual subcategory,
@@ -261,7 +264,7 @@ static bool subcateg_has_songid(uint32_t songid, subcategory_s* subcateg)
     return false;
 }
 
-static void add_song_to_subcateg(uint32_t songid, subcategory_s* subcateg)
+void add_song_to_subcateg(uint32_t songid, subcategory_s* subcateg)
 {
     if ( is_a_custom(songid)
      && !subcateg_has_songid(songid, subcateg) )
@@ -279,7 +282,7 @@ static void add_song_to_subcateg(uint32_t songid, subcategory_s* subcateg)
     }
 }
 
-static subcategory_s* get_subcateg(const char *title)
+subcategory_s* get_subcateg(const char *title)
 {
     for (uint32_t i = 0; i < g_subcateg_count; i++)
     {
@@ -287,7 +290,15 @@ static subcategory_s* get_subcateg(const char *title)
         if (strcmp(title, subcategories[i+1].name) == 0)
             return &(subcategories[i+1]);
     }
-    return NULL;
+
+    //not found, allocate a new one
+    subcategories = (subcategory_s*)realloc(subcategories, sizeof(subcategory_s)*((++g_subcateg_count)+1));
+    subcategory_s *subcateg = &(subcategories[g_subcateg_count]);
+    subcateg->name = strdup(title);
+    subcateg->songlist = NULL;
+    subcateg->size = 0;
+
+    return subcateg;
 }
 
 void (*categ_inject_songlist)();
@@ -506,6 +517,7 @@ void hook_categ_reinit_songlist()
     real_categ_reinit_songlist();
 }
 
+// in simple category mode, the game itself builds the full list by checking songids on the fly
 void (*real_categ_build_songlist)();
 void hook_categ_build_songlist()
 {
@@ -760,7 +772,7 @@ static bool patch_favorite_categ(const char *game_dll_fn) {
     return true;
 }
 
-static bool patch_custom_categ(const char *game_dll_fn) {
+static bool patch_custom_categ(const char *game_dll_fn, uint16_t min_id) {
 
     DWORD dllSize = 0;
     char *data = getDllData(game_dll_fn, &dllSize);
@@ -905,77 +917,21 @@ static bool patch_custom_categ(const char *game_dll_fn) {
 
 char formatted_title[128];
 sprintf(formatted_title, g_categformat, g_categname);
-    LOG("popnhax: custom %s \"%s\" injected (for songids ", g_subcategmode? "subcategories":"category", formatted_title);
-    if (g_max_id)
-        LOG("between %d and %d (incl.))\n", g_min_id, g_max_id);
-    else
-        LOG("%d and up)\n", g_min_id);
+    LOG("popnhax: custom %s \"%s\" injected", g_subcategmode? "subcategories":"category", formatted_title);
+    if (min_id)
+        LOG(" (for songids >= %d)", min_id);
+    LOG("\n");
 
     return true;
 }
 
-//extract folder name (cut "data_mods")
-static char *get_folder_name(const char* path) {
-    size_t len = (size_t)(strchr(path+10, '\\')-(path+10));
-    char *categ_name = (char*) malloc(len+1);
-    strncpy(categ_name, path+10, len);
-    categ_name[len] = '\0';
-    return categ_name;
-}
-
-
-bool is_excluded_folder(const char *input_filename)
-{
-  return (input_filename[strlen("data_mods/")] == '_');
-}
-
-static void parse_musicdb(const char *input_filename) {
-    char *title = get_folder_name(input_filename);
-
-    subcategory_s *subcateg = get_subcateg(title);
-    if ( subcateg == NULL )
-    {
-        subcategories = (subcategory_s*)realloc(subcategories, sizeof(subcategory_s)*((++g_subcateg_count)+1));
-        subcateg = &(subcategories[g_subcateg_count]);
-        subcateg->name = strdup(title);
-        subcateg->songlist = NULL;
-        subcateg->size = 0;
-    }
-
-    property* musicdb = load_prop_file(input_filename);
-    //iterate over all music id
-    property_node *prop = NULL;
-    if ((prop = property_search(musicdb, NULL, "/database/music"))) {
-        for (; prop != NULL; prop = property_node_traversal(prop, TRAVERSE_NEXT_SEARCH_RESULT)) {
-            char idxStr[256] = {};
-            property_node_refer(musicdb, prop, "id@", PROPERTY_TYPE_ATTR, idxStr,
-                                sizeof(idxStr));
-            uint32_t songid = atoi(idxStr);
-            add_song_to_subcateg(songid, subcateg);
-        }
-    }
-    free(title);
-}
-
-static void load_databases() {
-    SearchFile s;
+void init_subcategories() {
+    g_subcategmode = true;
     g_subcateg_count = 0;
     subcategories = (subcategory_s*)realloc(subcategories, sizeof(subcategory_s)*(1));
     subcategories[0].name = strdup("ALL SONGS");
     subcategories[0].songlist = NULL;
     subcategories[0].size = 0;
-
-    s.search("data_mods", "xml", true);
-    auto result = s.getResult();
-
-    for(uint16_t i=0;i<result.size();i++)
-    {
-        if ( (strstr(result[i].c_str(), "musicdb") == NULL)
-          || is_excluded_folder(result[i].c_str()) )
-            continue;
-
-        parse_musicdb(result[i].c_str());
-    }
 }
 
 static void print_databases() {
@@ -997,7 +953,7 @@ void hook_after_getlevel()
     __asm("push ecx\n");
     __asm("push edx\n");
     __asm("push ebx\n");
-    __asm("call %P0" : : "i"(is_a_custom));
+    __asm("call %P0" : : "i"(is_excluded_from_level));
     __asm("test eax, eax\n");
     __asm("pop ebx\n");
     __asm("pop edx\n");
@@ -1038,8 +994,6 @@ bool patch_exclude(const char *game_dll_fn)
 
 bool patch_custom_categs(const char *dllFilename, struct popnhax_config *config)
 {
-    g_min_id = config->custom_categ_min_songid;
-    //g_max_id = config->custom_categ_max_songid; //handled during injection already
     uint8_t mode = config->custom_categ;
 
     char icon_path[64];
@@ -1054,11 +1008,8 @@ bool patch_custom_categs(const char *dllFilename, struct popnhax_config *config)
         g_categicon = "cate_13";
     }
 
-    if (mode == 2)
+    if (mode == 1)
     {
-        g_subcategmode = true;
-        load_databases();
-    } else {
         songlist = (uint32_t*)calloc(1,5); //for some reason it crashes without that 4 cells extra margin
     }
 
@@ -1071,7 +1022,7 @@ bool patch_custom_categs(const char *dllFilename, struct popnhax_config *config)
     else
         g_categformat = "%s";
 
-    if (!patch_custom_categ(dllFilename))
+    if (!patch_custom_categ(dllFilename, config->custom_categ_min_songid))
         return false;
 
     if (mode == 2)
@@ -1089,8 +1040,12 @@ bool patch_custom_categs(const char *dllFilename, struct popnhax_config *config)
 
     if (config->custom_exclude_from_version)
         LOG("popnhax: Customs excluded from version folders\n"); //musichax_core_init took care of it
+
     if (config->custom_exclude_from_level)
+    {
+        g_exclude_omni = config->exclude_omni;
         patch_exclude(dllFilename);
+    }
 
     return true;
 }

popnhax/custom_categs.h

@@ -3,10 +3,21 @@
 
 #include <stdint.h>
 #include "popnhax/config.h"
+#include "util/bst.h"
+
+typedef struct {
+    char *name;
+    uint32_t size;
+    uint32_t *songlist; //really is a (songlist_t *) pointer
+} subcategory_s;
 
 extern uint32_t g_max_id;
+extern bst_t *g_customs_bst;
+
+void init_subcategories();
+void add_song_to_subcateg(uint32_t songid, subcategory_s* subcateg);
+subcategory_s* get_subcateg(const char *title);
 
-bool is_excluded_folder(const char *input_filename);
 bool patch_custom_categs(const char *dllFilename, struct popnhax_config *config);
 bool patch_local_favorites(const char *dllFilename, uint8_t version);
 

popnhax/dllmain.cc

@@ -187,10 +187,12 @@ PSMAP_MEMBER_REQ(PSMAP_PROPERTY_TYPE_S8, struct popnhax_config, base_offset,
                  "/popnhax/base_offset")
 PSMAP_MEMBER_REQ(PSMAP_PROPERTY_TYPE_U8, struct popnhax_config, custom_categ,
                  "/popnhax/custom_categ")
+PSMAP_MEMBER_REQ(PSMAP_PROPERTY_TYPE_BOOL, struct popnhax_config, exclude_omni,
+                 "/popnhax/exclude_omni")
+PSMAP_MEMBER_REQ(PSMAP_PROPERTY_TYPE_BOOL, struct popnhax_config, partial_entries,
+                 "/popnhax/partial_entries")
 PSMAP_MEMBER_REQ(PSMAP_PROPERTY_TYPE_U16, struct popnhax_config, custom_categ_min_songid,
                  "/popnhax/custom_categ_min_songid")
-PSMAP_MEMBER_REQ(PSMAP_PROPERTY_TYPE_U16, struct popnhax_config, custom_categ_max_songid,
-                 "/popnhax/custom_categ_max_songid")
 PSMAP_MEMBER_REQ(PSMAP_PROPERTY_TYPE_BOOL, struct popnhax_config, custom_exclude_from_version,
                  "/popnhax/custom_exclude_from_version")
 PSMAP_MEMBER_REQ(PSMAP_PROPERTY_TYPE_BOOL, struct popnhax_config, custom_exclude_from_level,

popnhax/loader.cc

@@ -4,6 +4,7 @@
 #include <windows.h>
 
 #include "imports/avs.h"
+#include "util/bst.h"
 #include "util/patch.h"
 #include "util/log.h"
 #include "xmlhelper.h"
@@ -820,12 +821,34 @@ void parse_charadb(const char *input_filename, const char *target) {
     free(config_xml);
 }
 
+//extract folder name (cut "data_mods")
+static char *get_folder_name(const char* path) {
+    size_t len = (size_t)(strchr(path+10, '\\')-(path+10));
+    char *categ_name = (char*) malloc(len+1);
+    strncpy(categ_name, path+10, len);
+    categ_name[len] = '\0';
+    return categ_name;
+}
+
+static bool is_excluded_folder(const char *input_filename)
+{
+  return (input_filename[strlen("data_mods/")] == '_');
+}
+
 void parse_musicdb(const char *input_filename, const char *target, struct popnhax_config *config) {
     if (!file_exists(input_filename)) {
         printf("Couldn't find %s, skipping...\n", input_filename);
         return;
     }
 
+    char *subcateg_title = NULL;
+    subcategory_s *subcateg = NULL;
+    if (config->custom_categ == 2)
+    {
+        subcateg_title = get_folder_name(input_filename);
+        subcateg = get_subcateg(subcateg_title); //will return a new one if not found
+    }
+
     property *config_xml = load_prop_file(input_filename);
 
     if (target && strlen(target) > 0) {
@@ -856,16 +879,29 @@ void parse_musicdb(const char *input_filename, const char *target, struct popnha
                                 sizeof(idxStr));
             uint32_t idx = atoi(idxStr);
 
-if (idx > g_max_id)
-{
-    g_max_id = idx;
-}
             // Get an existing music entry in memory
             // If it exists, return the existing entry
             // If it doesn't exist, create a new entry in memory
             // Update the data in-place and make all parameters optional
             music_entry *m = get_music(idx);
-            bool is_fresh = m == NULL;
+            bool is_fresh = m == NULL; // ie. not part of internal songdb
+            bool is_gone = ( m != NULL && strcmp((const char*) m->title_ptr, "\x81\x5D") == 0); // removed entries all have this title (SJIS "-")
+
+            // Update customs/omni songid list
+            if ( is_fresh || is_gone || config->partial_entries )
+            {
+                if ( idx >= config->custom_categ_min_songid && bst_search(g_customs_bst, idx) == NULL )
+                {
+                    g_customs_bst = bst_insert(g_customs_bst, idx);
+                    //LOG("%d inserted into customs bst\n", idx);
+                    if (config->custom_categ == 2)
+                    {
+                        add_song_to_subcateg(idx, subcateg);
+                    }
+                } else {
+                    //LOG("%d already present in customs bst\n", idx);
+                }
+            }
 
             if (is_fresh) {
                 // Default music entry
@@ -921,14 +957,11 @@ if (idx > g_max_id)
                 }
             }
 
-            //force loading background for unilab
-            //m->mask |= 0x100;
-
             if ( config->custom_categ
               && config->custom_exclude_from_version
               && !is_excluded_folder(input_filename)
-              && idx >= config->custom_categ_min_songid
-              && (config->custom_categ_max_songid == 0 || idx <= config->custom_categ_max_songid) )
+              && idx >= config->custom_categ_min_songid 
+              && ( is_fresh || config->exclude_omni ) )
             {
                 m->cs_version = 0;
                 m->folder = 0;
@@ -1009,6 +1042,11 @@ if (idx > g_max_id)
     }
 
     free(config_xml);
+
+    if (config->custom_categ == 2)
+    {
+        free(subcateg_title);
+    }
 }
 
 void load_databases(const char *target_datecode, struct popnhax_config *config) {
@@ -1027,6 +1065,9 @@ void load_databases(const char *target_datecode, struct popnhax_config *config)
         parse_charadb(result[i].c_str(), target_datecode);
     }
 
+    if (config->custom_categ == 2)
+        init_subcategories();
+
     for(uint16_t i=0;i<result.size();i++)
     {
         if ( strstr(result[i].c_str(), "musicdb") == NULL )

util/Module.mk

@@ -2,6 +2,7 @@ libs                    += util
 
 srcpp_util              := \
 	fuzzy_search.cc \
+	bst.cc \
 	search.cc \
 	cmdline.cc \
 	patch.cc \

util/bst.cc

@@ -0,0 +1,34 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+
+#include "bst.h"
+
+bst_t* bst_search(bst_t *root, uint32_t val)
+{
+    if( root == NULL || root->data == val )
+        return root;
+    else if( val > (root->data) )
+        return bst_search(root->right, val);
+    else
+        return bst_search(root->left,val);
+}
+
+bst_t* bst_insert(bst_t *root, uint32_t val)
+{
+    if ( root == NULL )
+    {
+        bst_t *p;
+        p = (bst_t *)malloc(sizeof(bst_t));
+        p->data = val;
+        p->left = NULL;
+        p->right = NULL;
+        return p;
+    }
+    else if ( val > root->data )
+        root->right = bst_insert(root->right, val);
+    else
+        root->left = bst_insert(root->left, val);
+
+    return root;
+}

util/bst.h

@@ -0,0 +1,14 @@
+#ifndef __BST_H__
+#define __BST_H__
+
+typedef struct bst_s
+{
+    uint32_t data;
+    struct bst_s *right;
+    struct bst_s *left;
+} bst_t;
+
+bst_t* bst_search(bst_t *root, uint32_t val);
+bst_t* bst_insert(bst_t *root, uint32_t val);
+
+#endif