mirror of
https://github.com/vichan-devel/vichan.git
synced 2024-11-24 07:30:10 +01:00
New CLI tool for upgrade: tools/migrate_board_stats.php
- Adds new coumn to `boards` for easy-access post total storage. - Migrates the AUTO_INCREMENT number to the `posts_total` column for every board. This ensures accurate post count measurement. - Adds `board_stats` table for recording histric data. - Retroactively records all boards into `board_stats`. - inc/functions.php Added handling for `posts_total` column in `boards`. - inc/functions.php Removed some bogus data from fetchBoardActivity - board-search.php Fixed issue with SFW filtering. - board-search.php Now accurately sends `posts_total` data for board sum posts. - boards-search.html Various form fixes so search now submits correctly. - boards-table.html Fixed `posts_total` value. Signed-off-by: 8n-tech <8n-tech@users.noreply.github.com>
This commit is contained in:
8n-tech
parent
6d1eb9961d
commit
316e681bbd
@ -23,26 +23,30 @@ $languages = array(
|
||||
|
||||
/* Determine search parameters from $_GET */
|
||||
$search = array(
|
||||
'lang' => false,
|
||||
'nsfw' => true,
|
||||
'tags' => false,
|
||||
'titles' => false,
|
||||
'lang' => false,
|
||||
'nsfw' => true,
|
||||
'tags' => false,
|
||||
'title' => false,
|
||||
);
|
||||
|
||||
// Include NSFW boards?
|
||||
if (isset( $_GET['nsfw'] )) {
|
||||
$search['nsfw'] = (bool) $_GET['nsfw'];
|
||||
if (isset( $_GET['sfw'] ) && $_GET['sfw'] != "") {
|
||||
$search['nsfw'] = !$_GET['sfw'];
|
||||
}
|
||||
|
||||
// Include what language (if the language is not blank and we recognize it)?
|
||||
if (isset( $_GET['lang'] ) && isset($languages[$search['lang']])) {
|
||||
if (isset( $_GET['lang'] ) && $_GET['lang'] != "" && isset($languages[$search['lang']])) {
|
||||
$search['lang'] = $_GET['lang'];
|
||||
}
|
||||
|
||||
// Include what tag?
|
||||
if (isset( $_GET['tags'] )) {
|
||||
if (isset( $_GET['tags'] ) && $_GET['tags'] != "") {
|
||||
$search['tags'] = $_GET['tags'];
|
||||
}
|
||||
// Include what in the uri / title / subtitle?
|
||||
if (isset( $_GET['title'] ) && $_GET['title'] != "") {
|
||||
$search['title'] = $_GET['title'];
|
||||
}
|
||||
|
||||
/* Search boards */
|
||||
$boards = listBoards();
|
||||
@ -108,7 +112,6 @@ $response['tags'] = array();
|
||||
// We will also be weighing and building a tag list.
|
||||
foreach ($response['boards'] as $boardUri => &$board) {
|
||||
$board['active'] = (int) $boardActivity['active'][ $boardUri ];
|
||||
$board['posts'] = (int) $boardActivity['posts'][ $boardUri ];
|
||||
$board['pph'] = (int) $boardActivity['average'][ $boardUri ];
|
||||
|
||||
if (isset($board['tags']) && count($board['tags']) > 0) {
|
||||
@ -124,13 +127,19 @@ foreach ($response['boards'] as $boardUri => &$board) {
|
||||
}
|
||||
|
||||
// Sort boards by their popularity, then by their total posts.
|
||||
$boardActivityValues = array();
|
||||
$boardActivityValues = array();
|
||||
$boardTotalPostsValues = array();
|
||||
|
||||
foreach ($response['boards'] as $boardUri => $board) {
|
||||
$boardActivityValues[$boardUri] = "{$board['active']}.{$board['posts']}";
|
||||
foreach ($response['boards'] as $boardUri => &$board) {
|
||||
$boardActivityValues[$boardUri] = (int) $board['active'];
|
||||
$boardTotalPostsValues[$boardUri] = (int) $board['posts_total'];
|
||||
}
|
||||
|
||||
array_multisort($boardActivityValues, SORT_DESC, $response['boards']);
|
||||
array_multisort(
|
||||
$boardActivityValues, SORT_DESC, SORT_NUMERIC, // Sort by number of active posters
|
||||
$boardTotalPostsValues, SORT_DESC, SORT_NUMERIC, // Then, sort by total number of posts
|
||||
$response['boards']
|
||||
);
|
||||
|
||||
// Get the top most popular tags.
|
||||
if (count($response['tags']) > 0) {
|
||||
@ -146,7 +155,7 @@ if (count($response['tags']) > 0) {
|
||||
/* (Please) Respond */
|
||||
if (!$Included) {
|
||||
$json = json_encode( $response );
|
||||
|
||||
|
||||
// Error Handling
|
||||
switch (json_last_error()) {
|
||||
case JSON_ERROR_NONE:
|
||||
@ -171,11 +180,11 @@ if (!$Included) {
|
||||
$jsonError = 'Unknown error';
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
if ($jsonError) {
|
||||
$json = "{\"error\":\"{$jsonError}\"}";
|
||||
}
|
||||
|
||||
|
||||
// Successful output
|
||||
echo $json;
|
||||
}
|
||||
|
||||
@ -786,9 +786,23 @@ function listBoards($just_uri = false, $indexed_only = false) {
|
||||
return $boards;
|
||||
|
||||
if (!$just_uri) {
|
||||
$query = query("SELECT ``boards``.`uri` uri, ``boards``.`title` title, ``boards``.`subtitle` subtitle, ``board_create``.`time` time, ``boards``.`indexed` indexed, ``boards``.`sfw` sfw FROM ``boards``" . ( $indexed_only ? " WHERE `indexed` = 1 " : "" ) . "LEFT JOIN ``board_create`` ON ``boards``.`uri` = ``board_create``.`uri` ORDER BY ``boards``.`uri`") or error(db_error());
|
||||
$query = query(
|
||||
"SELECT
|
||||
``boards``.`uri` uri,
|
||||
``boards``.`title` title,
|
||||
``boards``.`subtitle` subtitle,
|
||||
``board_create``.`time` time,
|
||||
``boards``.`indexed` indexed,
|
||||
``boards``.`sfw` sfw,
|
||||
``boards``.`posts_total` posts_total
|
||||
FROM ``boards``" . ( $indexed_only ? " WHERE `indexed` = 1 " : "" ) .
|
||||
"LEFT JOIN ``board_create``
|
||||
ON ``boards``.`uri` = ``board_create``.`uri`
|
||||
ORDER BY ``boards``.`uri`") or error(db_error());
|
||||
|
||||
$boards = $query->fetchAll(PDO::FETCH_ASSOC);
|
||||
} else {
|
||||
}
|
||||
else {
|
||||
$boards = array();
|
||||
$query = query("SELECT `uri` FROM ``boards``" . ( $indexed_only ? " WHERE `indexed` = 1" : "" ) . " ORDER BY ``boards``.`uri`") or error(db_error());
|
||||
while (true) {
|
||||
@ -823,45 +837,14 @@ function fetchBoardActivity( $uris ) {
|
||||
global $config;
|
||||
|
||||
$boardActivity = array();
|
||||
//$uris = "\"" . implode( (array) $uris, "\",\"" ) . "\"";
|
||||
|
||||
$tablePrefix = "{$config['db']['prefix']}posts_";
|
||||
$uris = "\"{$tablePrefix}" . implode( (array) $uris, "\",\"{$tablePrefix}" ) . "\"";
|
||||
|
||||
/*
|
||||
$tagQuery = prepare("SELECT * FROM ``board_tags`` WHERE `uri` IN ({$uris})");
|
||||
$tagQuery->execute() or error(db_error($tagQuery));
|
||||
$tagResult = $tagQuery->fetchAll(PDO::FETCH_ASSOC);
|
||||
|
||||
if ($tagResult) {
|
||||
foreach ($tagResult as $tagRow) {
|
||||
$tag = $tagRow['tag'];
|
||||
$tag = trim($tag);
|
||||
$tag = strtolower($tag);
|
||||
$tag = str_replace(['_', ' '], '-', $tag);
|
||||
|
||||
if (!isset($boardTags[ $tagRow['uri'] ])) {
|
||||
$boardTags[ $tagRow['uri'] ] = array();
|
||||
}
|
||||
|
||||
$boardTags[ $tagRow['uri'] ][] = htmlentities( utf8_encode( $tag ) );
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
$aiQuery = prepare("SELECT `TABLE_NAME`, `AUTO_INCREMENT` FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = \"{$config['db']['database']}\" AND TABLE_NAME IN ({$uris})");
|
||||
$aiQuery->execute() or error(db_error($aiQuery));
|
||||
$aiResult = $aiQuery->fetchAll(PDO::FETCH_ASSOC);
|
||||
|
||||
foreach ($aiResult as $aiRow) {
|
||||
$uri = str_replace( $tablePrefix, "", $aiRow['TABLE_NAME'] );
|
||||
$posts = $aiRow['AUTO_INCREMENT'] - 1;
|
||||
|
||||
$random = rand( -1000, 1000 );
|
||||
foreach ($uris as $uri) {
|
||||
$random = 0;//rand( -1000, 1000 );
|
||||
if( $random < 0 ) $random = 0;
|
||||
|
||||
$boardActivity['active'][ $uri ] = $random;
|
||||
$boardActivity['average'][ $uri ] = ($random * 72) / 72;
|
||||
$boardActivity['posts'][ $uri ] = $posts;
|
||||
}
|
||||
|
||||
return $boardActivity;
|
||||
|
||||
@ -1,40 +1,40 @@
|
||||
<main id="boardlist">
|
||||
<section class="description box col col-12">
|
||||
<h2 class="box-title">Global Statistics</h2>
|
||||
<p class="box-content">{% trans %}There are currently <strong>{{boards_public}}</strong> public boards, <strong>{{boards_total}}</strong> total. Site-wide, {{posts_hour}} posts have been made in the last hour, with {{posts_total}} being made on all active boards since {{founding_date}}.{% endtrans %}</p>
|
||||
<p class="box-content">{% trans %}There are currently <strong>{{boards_public}}</strong> public boards, <strong>{{boards_total}}</strong> total. Site-wide, <strong>{{posts_hour}}</strong> posts have been made in the last hour, with <strong>{{posts_total}}</strong> being made on all active boards since {{founding_date}}.{% endtrans %}</p>
|
||||
{% if uptime %}<p class="box-content">{{uptime}} without interruption</p>{% endif %}
|
||||
<p class="box-content">This page last updated {{page_updated}}.</p>
|
||||
</section>
|
||||
|
||||
<div class="board-list">
|
||||
<aside class="search-container col col-2">
|
||||
<form id="search-form" class="box" method="post" target="/board-search.php">
|
||||
<form id="search-form" class="box" method="get" target="/board-search.php">
|
||||
<h2 class="box-title">Search</h2>
|
||||
|
||||
<div class="board-search box-content">
|
||||
<label class="search-item search-sfw">
|
||||
<input type="checkbox" id="search-sfw-input" name="sfw" checked="checked" /> NSFW boards
|
||||
<input type="checkbox" id="search-sfw-input" name="sfw" value="1" /> Hide NSFW boards
|
||||
</label>
|
||||
|
||||
<div class="search-item search-title">
|
||||
<input type="text" id="search-title-input" name="title" placeholder="Search titles..." />
|
||||
<input type="text" id="search-title-input" name="title" name="title" value="" placeholder="Search titles..." />
|
||||
</div>
|
||||
|
||||
<div class="search-item search-lang">
|
||||
<select id="search-lang-input" name="lang">
|
||||
<optgroup label="Popular">
|
||||
<option>All languages</option>
|
||||
<option>English</option>
|
||||
<option>Spanish</option>
|
||||
<option value="">All languages</option>
|
||||
<option value="en">English</option>
|
||||
<option value="es">Spanish</option>
|
||||
</optgroup>
|
||||
<optgroup label="All">
|
||||
<option>Chinese</option>
|
||||
<option value="cn">Chinese</option>
|
||||
</optgroup>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="search-item search-tag">
|
||||
<input type="text" id="search-tag-input" name="tag" placeholder="Search tags..." />
|
||||
<input type="text" id="search-tag-input" name="tags" value="" placeholder="Search tags..." />
|
||||
</div>
|
||||
|
||||
<div class="search-item search-submit">
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
<td class="board-uri"><div class="board-list-wrapper uri"><a href='/{{board['uri']}}/'>/{{board['uri']}}/</a>{{lock|raw}}</div></td>
|
||||
<td class="board-title"><div class="board-cell" title="Created {{board['time']}} ({{board['ago']}} ago)">{{ board['title'] }}</div></td>
|
||||
<td class="board-pph"><div class="board-cell">{{board['pph']}}</td>
|
||||
<td class="board-max"><div class="board-cell">{{board['posts']}}</td>
|
||||
<td class="board-max"><div class="board-cell">{{board['posts_total']}}</td>
|
||||
<td class="board-unique"><div class="board-cell">{{board['active']}}</td>
|
||||
<td class="board-tags"><div class="board-cell">{% for tag in board.tags %}<span class="board-tag">{{ tag }}</span> {% endfor %}</div></td>
|
||||
</tr>
|
||||
|
||||
125
tools/migrate_board_stats.php
Normal file
125
tools/migrate_board_stats.php
Normal file
@ -0,0 +1,125 @@
|
||||
<?php
|
||||
require dirname(__FILE__) . '/inc/cli.php';
|
||||
|
||||
/* Convert AI value to colun value for ez access */
|
||||
// Add column `posts_total` to `boards`.
|
||||
// This can potentially error if ran multiple times.. but that shouldn't kill the script
|
||||
echo "Altering `boards` to add `posts_total`...\n";
|
||||
query( "ALTER TABLE `boards` ADD COLUMN `posts_total` INT(11) UNSIGNED NOT NULL DEFAULT 0" );
|
||||
|
||||
// Set the value for posts_total for each board.
|
||||
echo "Updating `boards` to include `posts_total` values...\n";
|
||||
$tablePrefix = "{$config['db']['prefix']}posts_";
|
||||
|
||||
$aiQuery = prepare("SELECT `TABLE_NAME`, `AUTO_INCREMENT` FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = \"{$config['db']['database']}\"");
|
||||
$aiQuery->execute() or error(db_error($aiQuery));
|
||||
$aiResult = $aiQuery->fetchAll(PDO::FETCH_ASSOC);
|
||||
|
||||
foreach ($aiResult as $aiRow) {
|
||||
$uri = str_replace( $tablePrefix, "", $aiRow['TABLE_NAME'] );
|
||||
$posts = (int)($aiRow['AUTO_INCREMENT'] - 1); // Don't worry! The column is unsigned. -1 becomes 0.
|
||||
|
||||
echo " {$uri} has {$posts} post".($posts!=1?"s":"")."\n";
|
||||
query( "UPDATE `boards` SET `posts_total`={$posts} WHERE `uri`=\"{$uri}\";" );
|
||||
}
|
||||
|
||||
unset( $aiQuery, $aiResult, $uri, $posts );
|
||||
|
||||
/* Add statistics table and transmute post information to that */
|
||||
// Add `board_stats`
|
||||
echo "Adding `board_stats` ...\n";
|
||||
query(
|
||||
"CREATE TABLE IF NOT EXISTS ``board_stats`` (
|
||||
`stat_uri` VARCHAR(58) NOT NULL,
|
||||
`stat_hour` INT(11) UNSIGNED NOT NULL,
|
||||
`post_count` INT(11) UNSIGNED NULL,
|
||||
`post_id_array` TEXT NULL,
|
||||
`author_ip_count` INT(11) UNSIGNED NULL,
|
||||
`author_ip_array` TEXT NULL,
|
||||
PRIMARY KEY (`stat_uri`, `stat_hour`)
|
||||
);"
|
||||
);
|
||||
|
||||
$boards = listBoards();
|
||||
|
||||
echo "Translating posts to stats ...\n";
|
||||
foreach ($boards as $board) {
|
||||
$postQuery = prepare("SELECT `id`, `time`, `ip` FROM ``posts_{$board['uri']}``");
|
||||
$postQuery->execute() or error(db_error($postQuery));
|
||||
$postResult = $postQuery->fetchAll(PDO::FETCH_ASSOC);
|
||||
|
||||
// Determine the number of posts for each hour.
|
||||
$postHour = array();
|
||||
|
||||
foreach ($postResult as $post) {
|
||||
// Winds back timestamp to last hour. (1428947438 -> 1428944400)
|
||||
$postHourTime = (int)($post['time'] / 3600) * 3600;
|
||||
|
||||
if (!isset($postHour[ $postHourTime ])) {
|
||||
$postHour[ $postHourTime ] = array();
|
||||
}
|
||||
|
||||
$postDatum = &$postHour[ $postHourTime ];
|
||||
|
||||
// Add to post count.
|
||||
if (!isset($postDatum['post_count'])) {
|
||||
$postDatum['post_count'] = 1;
|
||||
}
|
||||
else {
|
||||
++$postDatum['post_count'];
|
||||
}
|
||||
|
||||
// Add to post id array.
|
||||
if (!isset($postDatum['post_id_array'])) {
|
||||
$postDatum['post_id_array'] = array( (int)$post['id'] );
|
||||
}
|
||||
else {
|
||||
$postDatum['post_id_array'][] = (int)$post['id'];
|
||||
}
|
||||
|
||||
// Add to ip array.
|
||||
if (!isset($postDatum['author_ip_array'])) {
|
||||
$postDatum['author_ip_array'][ less_ip( $post['ip'] ) ] = 1;
|
||||
}
|
||||
|
||||
// Count ip array.
|
||||
$postDatum['author_ip_count'] = count( $postDatum['post_id_array'] );
|
||||
|
||||
unset( $postHourTime );
|
||||
}
|
||||
|
||||
// Prep data for insert.
|
||||
foreach ($postHour as $postHourTime => &$postHourData) {
|
||||
$postDatum = &$postHour[ $postHourTime ];
|
||||
|
||||
// Serialize arrays for TEXT insert.
|
||||
$postDatum['post_id_array' ] = str_replace( "\"", "\\\"", serialize( $postDatum['post_id_array'] ) );
|
||||
$postDatum['author_ip_array'] = str_replace( "\"", "\\\"", serialize( array_keys( $postDatum['author_ip_array'] ) ) );
|
||||
}
|
||||
|
||||
// Bash this shit together into a set of insert statements.
|
||||
$statsInserts = array();
|
||||
|
||||
foreach ($postHour as $postHourTime => $postHourData) {
|
||||
$statsInserts[] = "(\"{$board['uri']}\", \"{$postHourTime}\", \"{$postHourData['post_count']}\", \"{$postHourData['post_id_array']}\", \"{$postHourData['author_ip_count']}\", \"{$postHourData['author_ip_array']}\" )";
|
||||
}
|
||||
|
||||
if (count($statsInserts) > 0) {
|
||||
$statsInsert = "VALUES" . implode( ", ", $statsInserts );
|
||||
echo " {$board['uri']} is building " . count($statsInserts) . " stat rows.\n";
|
||||
|
||||
// Insert this data into our statistics table.
|
||||
$postStatQuery = prepare(
|
||||
"REPLACE INTO ``board_stats`` (stat_uri, stat_hour, post_count, post_id_array, author_ip_count, author_ip_array) {$statsInsert}"
|
||||
);
|
||||
$postStatQuery->execute() or error(db_error($postStatQuery));
|
||||
}
|
||||
else {
|
||||
echo " {$board['uri']} has no posts!\n";
|
||||
}
|
||||
|
||||
unset( $postQuery, $postResult, $postStatQuery, $postHour, $statsInserts, $statsInsert );
|
||||
}
|
||||
|
||||
|
||||
echo "Done! ^^;";
|
||||
Loading…
Reference in New Issue
Block a user