Cool_Tools/vichan
1
0
Fork 0
mirror of https://github.com/vichan-devel/vichan.git synced 2024-11-23 23:20:57 +01:00
Code Releases Wiki Activity

advanced build (1/2): a small refactor of index generating procedure; generation strategies

Browse Source
This commit is contained in:
czaks 2016-05-08 10:54:30 +02:00
parent 644f227ab3
commit b6f0317bde
8 changed files with 169 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
inc/config.php
View File

@ -1203,16 +1203,74 @@
// Try not to build pages when we shouldn't have to.
$config['try_smarter'] = true;
// EXPERIMENTAL: Defer static HTML building to a moment, when a given file is actually accessed.
// Warning: This option won't run out of the box. You need to tell your webserver, that a file
// for serving 403 and 404 pages is /smart_build.php. Also, you need to turn off indexes.
/*
* ====================
* Advanced build
* ====================
*/
// Strategies for file generation. Also known as an "advanced build". If you don't have performance
// issues, you can safely ignore that part, because it's hard to configure and won't even work on
// your free webhosting.
//
// A strategy is a function, that given the PHP environment and ($fun, $array) variable pair, returns
// an $action array or false.
//
// $fun - a controller function name, see inc/controller.php. This is named after functions, so that
// we can generate the files in daemon.
//
// $array - arguments to be passed
//
// $action - action to be taken. It's an array, and the first element of it is one of the following:
// * "immediate" - generate the page immediately
// * "defer" - defer page generation to a moment a worker daemon gets to build it (serving a stale
// page in the meantime). The remaining arguments are daemon-specific. Daemon isn't
// implemented yet :DDDD inb4 while(true) { generate(Queue::Get()) }; (which is probably it).
// * "build_on_load" - defer page generation to a moment, when the user actually accesses the page.
// This is a smart_build behaviour. You shouldn't use this one too much, if you
// use it for active boards, the server may choke due to a possible race condition.
// See my blog post: https://engine.vichan.net/blog/res/2.html
//
// So, let's assume we want to build a thread 1324 on board /b/, because a new post appeared there.
// We try the first strategy, giving it arguments: 'sb_thread', array('b', 1324). The strategy will
// now return a value $action, denoting an action to do. If $action is false, we try another strategy.
//
// As I said, configuration isn't easy.
//
// 1. chmod 0777 directories: tmp/locks/ and tmp/queue/.
// 2. serve 403 and 404 requests to go thru smart_build.php
// for nginx, this blog post contains config snippets: https://engine.vichan.net/blog/res/2.html
// 3. disable indexes in your webserver
// 4. launch any number of daemons (eg. twice your number of threads?) using the command:
// $ tools/worker.php
// You don't need to do that step if you are not going to use the "defer" option.
// 5. enable smart_build_helper (see below)
// 6. edit the strategies (see inc/functions.php for the builtin ones). You can use lambdas. I will test
// various ones and include one that works best for me.
$config['generation_strategies'] = array();
// Add a sane strategy. It forces to immediately generate a page user is about to land on. Otherwise,
// it has no opinion, so it needs a fallback strategy.
$config['generation_strategies'][] = 'strategy_sane';
// Add an immediate catch-all strategy. This is the default function of imageboards: generate all pages
// on post time.
$config['generation_strategies'][] = 'strategy_immediate';
// NOT RECOMMENDED: Instead of an all-"immediate" strategy, you can use an all-"build_on_load" one (used
// to be initialized using $config['smart_build']; ) for all pages instead of those to be build
// immediately. A rebuild done in this mode should remove all your static files
// $config['generation_strategies'][1] = 'strategy_smart_build';
// Deprecated. Leave it false. See above.
$config['smart_build'] = false;
// Smart build related: when a file doesn't exist, where should we redirect?
// Use smart_build.php for dispatching missing requests. It may be useful without smart_build or advanced
// build, for example it will regenerate the missing files.
$config['smart_build_helper'] = true;
// smart_build.php: when a file doesn't exist, where should we redirect?
$config['page_404'] = '/404.html';
// Smart build related: extra entrypoints.
$config['smart_build_entrypoints'] = array();
// Extra controller entrypoints. Controller is used only by smart_build and advanced build.
$config['controller_entrypoints'] = array();
/*
* ====================

102
inc/functions.php
View File

@ -1319,7 +1319,8 @@ function thread_find_page($thread) {
return floor(($config['threads_per_page'] + $index) / $config['threads_per_page']);
}
function index($page, $mod=false) {
// $brief means that we won't need to generate anything yet
function index($page, $mod=false, $brief = false) {
global $board, $config, $debug;
$body = '';
@ -1350,6 +1351,7 @@ function index($page, $mod=false) {
unset($cached);
}
}
if (!isset($cached)) {
$posts = prepare(sprintf("SELECT * FROM ``posts_%s`` WHERE `thread` = :id ORDER BY `id` DESC LIMIT :limit", $board['uri']));
$posts->bindValue(':id', $th['id']);
@ -1389,7 +1391,10 @@ function index($page, $mod=false) {
}
$threads[] = $thread;
$body .= $thread->build(true);
if (!$brief) {
$body .= $thread->build(true);
}
}
if ($config['file_board']) {
@ -1610,27 +1615,28 @@ function checkMute() {
function buildIndex($global_api = "yes") {
global $board, $config, $build_pages;
if (!$config['smart_build']) {
$pages = getPages();
if (!$config['try_smarter'])
$antibot = create_antibot($board['uri']);
$catalog_api_action = generation_strategy('sb_api', array($board['uri']));
if ($config['api']['enabled']) {
$api = new Api();
$catalog = array();
}
$pages = null;
$antibot = null;
if ($config['api']['enabled']) {
$api = new Api();
$catalog = array();
}
for ($page = 1; $page <= $config['max_pages']; $page++) {
$filename = $board['dir'] . ($page == 1 ? $config['file_index'] : sprintf($config['file_page'], $page));
$jsonFilename = $board['dir'] . ($page - 1) . '.json'; // pages should start from 0
if ((!$config['api']['enabled'] || $global_api == "skip" || $config['smart_build']) && $config['try_smarter']
&& isset($build_pages) && !empty($build_pages) && !in_array($page, $build_pages) )
$wont_build_this_page = $config['try_smarter'] && isset($build_pages) && !empty($build_pages) && !in_array($page, $build_pages);
if ((!$config['api']['enabled'] || $global_api == "skip") && $wont_build_this_page)
continue;
if (!$config['smart_build']) {
$content = index($page);
$action = generation_strategy('sb_board', array($board['uri'], $page));
if ($action == 'rebuild' || $catalog_api_action == 'rebuild') {
$content = index($page, false, $wont_build_this_page);
if (!$content)
break;
@ -1641,17 +1647,21 @@ function buildIndex($global_api = "yes") {
file_write($jsonFilename, $json);
$catalog[$page-1] = $threads;
}
if ($config['api']['enabled'] && $global_api != "skip" && $config['try_smarter'] && isset($build_pages)
&& !empty($build_pages) && !in_array($page, $build_pages) )
continue;
if ($wont_build_this_page) continue;
}
if ($config['try_smarter']) {
$antibot = create_antibot($board['uri'], 0 - $page);
$content['current_page'] = $page;
}
elseif (!$antibot) {
create_antibot($board['uri']);
}
$antibot->reset();
if (!$pages) {
$pages = getPages();
}
$content['pages'] = $pages;
$content['pages'][$page-1]['selected'] = true;
$content['btn'] = getPageButtons($content['pages']);
@ -1659,13 +1669,14 @@ function buildIndex($global_api = "yes") {
file_write($filename, Element('index.html', $content));
}
else {
elseif ($action == 'delete' || $catalog_api_action == 'delete') {
file_unlink($filename);
file_unlink($jsonFilename);
}
}
if (!$config['smart_build'] && $page < $config['max_pages']) {
// $action is an action for our last page
if (($catalog_api_action == 'rebuild' || $action == 'rebuild' || $action == 'delete') && $page < $config['max_pages']) {
for (;$page<=$config['max_pages'];$page++) {
$filename = $board['dir'] . ($page==1 ? $config['file_index'] : sprintf($config['file_page'], $page));
file_unlink($filename);
@ -1679,13 +1690,13 @@ function buildIndex($global_api = "yes") {
// json api catalog
if ($config['api']['enabled'] && $global_api != "skip") {
if ($config['smart_build']) {
if ($catalog_api_action == 'delete') {
$jsonFilename = $board['dir'] . 'catalog.json';
file_unlink($jsonFilename);
$jsonFilename = $board['dir'] . 'threads.json';
file_unlink($jsonFilename);
}
else {
elseif ($catalog_api_action == 'rebuild') {
$json = json_encode($api->translateCatalog($catalog));
$jsonFilename = $board['dir'] . 'catalog.json';
file_write($jsonFilename, $json);
@ -2204,7 +2215,9 @@ function buildThread($id, $return = false, $mod = false) {
if ($config['try_smarter'] && !$mod)
$build_pages[] = thread_find_page($id);
if (!$config['smart_build'] || $return || $mod) {
$action = generation_strategy('sb_thread', array($board['uri'], $id));
if ($action == 'rebuild' || $return || $mod) {
$query = prepare(sprintf("SELECT * FROM ``posts_%s`` WHERE (`thread` IS NULL AND `id` = :id) OR `thread` = :id ORDER BY `thread`,`id`", $board['uri']));
$query->bindValue(':id', $id, PDO::PARAM_INT);
$query->execute() or error(db_error($query));
@ -2239,26 +2252,26 @@ function buildThread($id, $return = false, $mod = false) {
));
// json api
if ($config['api']['enabled']) {
if ($config['api']['enabled'] && !$mod) {
$api = new Api();
$json = json_encode($api->translateThread($thread));
$jsonFilename = $board['dir'] . $config['dir']['res'] . $id . '.json';
file_write($jsonFilename, $json);
}
}
else {
elseif($action == 'delete') {
$jsonFilename = $board['dir'] . $config['dir']['res'] . $id . '.json';
file_unlink($jsonFilename);
}
if ($config['smart_build'] && !$return && !$mod) {
if ($action == 'delete' && !$return && !$mod) {
$noko50fn = $board['dir'] . $config['dir']['res'] . link_for(array('id' => $id), true);
file_unlink($noko50fn);
file_unlink($board['dir'] . $config['dir']['res'] . link_for(array('id' => $id)));
} else if ($return) {
} elseif ($return) {
return $body;
} else {
} elseif ($action == 'rebuild') {
$noko50fn = $board['dir'] . $config['dir']['res'] . link_for($thread, true);
if ($hasnoko50 || file_exists($noko50fn)) {
buildThread50($id, $return, $mod, $thread, $antibot);
@ -2788,3 +2801,36 @@ function markdown($s) {
return $pd->text($s);
}
function generation_strategy($fun, $array=array()) { global $config;
$action = false;
foreach ($config['generation_strategies'] as $s) {
if ($strategy = $s($fun, $array)) {
break;
}
}
switch ($strategy[0]) {
case 'immediate':
return 'rebuild';
case 'defer':
// Ok, it gets interesting here :)
Queue::add(serialize(array('build', $fun, $array)));
return 'ignore';
case 'build_on_load':
return 'delete';
}
}
function strategy_immediate($fun, $array) {
return array('immediate');
}
function strategy_smart_build($fun, $array) {
return array('build_on_load');
}
function strategy_sane($fun, $array) { global $config;
return false;
}

5
inc/route.php
View File

@ -7,7 +7,7 @@
defined('TINYBOARD') or exit;
function route($path) {
function route($path) { global $config;
$entrypoints = array();
$entrypoints['/%b/'] = 'sb_board';
@ -33,8 +33,11 @@ function route($path) {
$entrypoints['/*/index.html'] = 'sb_ukko';
$entrypoints['/recent.html'] = 'sb_recent';
$entrypoints['/%b/catalog.html'] = 'sb_catalog';
$entrypoints['/%b/index.rss'] = 'sb_catalog';
$entrypoints['/sitemap.xml'] = 'sb_sitemap';
$entrypoints = array_merge($entrypoints, $config['controller_entrypoints']);
$reached = false;
list($request) = explode('?', $path);

9
smart_build.php
View File

@ -3,14 +3,16 @@ require_once("inc/functions.php");
require_once("inc/route.php");
require_once("inc/controller.php");
if (!$config['smart_build'] && !$config["smart_build_helper"]) {
die('You need to enable $config["smart_build"] or $config["smart_build_helper"]');
if (!$config["smart_build_helper"]) {
die('You need to enable $config["smart_build_helper"]');
}
$config['smart_build'] = false; // Let's disable it, so we can build the page for real
$config['generation_strategies'] = array('strategy_immediate');
function after_open_board() { global $config;
$config['smart_build'] = false;
$config['generation_strategies'] = array('strategy_immediate');
};
$request = $_SERVER['REQUEST_URI'];
@ -59,6 +61,9 @@ if ($reached) {
elseif (preg_match('/\.xml$/', $request)) {
header("Content-Type", "application/xml");
}
elseif (preg_match('/\.rss$/', $request)) {
header("Content-Type", "application/rss+xml");
}
else {
header("Content-Type", "text/html; charset=utf-8");
}

13
templates/themes/catalog/theme.php
View File

@ -16,20 +16,25 @@
if ($action == 'all') {
foreach ($boards as $board) {
$b = new Catalog();
if ($config['smart_build']) {
$action = generation_strategy("sb_catalog", array($board));
if ($action == 'delete') {
file_unlink($config['dir']['home'] . $board . '/catalog.html');
file_unlink($config['dir']['home'] . $board . '/index.rss');
}
else {
elseif ($action == 'rebuild') {
$b->build($settings, $board);
}
}
} elseif ($action == 'post-thread' || ($settings['update_on_posts'] && $action == 'post') || ($settings['update_on_posts'] && $action == 'post-delete') && in_array($board, $boards)) {
$b = new Catalog();
if ($config['smart_build']) {
$action = generation_strategy("sb_catalog", array($board));
if ($action == 'delete') {
file_unlink($config['dir']['home'] . $board . '/catalog.html');
file_unlink($config['dir']['home'] . $board . '/index.rss');
}
else {
elseif ($action == 'rebuild') {
$b->build($settings, $board);
}
}

5
templates/themes/recent/theme.php
View File

@ -25,10 +25,11 @@
$this->excluded = explode(' ', $settings['exclude']);
if ($action == 'all' || $action == 'post' || $action == 'post-thread' || $action == 'post-delete') {
if ($config['smart_build']) {
$action = generation_strategy('sb_recent', array());
if ($action == 'delete') {
file_unlink($config['dir']['home'] . $settings['html']);
}
else {
elseif ($action == 'rebuild') {
file_write($config['dir']['home'] . $settings['html'], $this->homepage($settings));
}
}

6
templates/themes/sitemap/theme.php
View File

@ -23,10 +23,12 @@
}
}
if ($config['smart_build']) {
$action = generation_strategy('sb_sitemap', array());
if ($action == 'delete') {
file_unlink($settings['path']);
}
else {
elseif ($action == 'rebuild') {
$boards = explode(' ', $settings['boards']);
$threads = array();

6
templates/themes/ukko/theme.php
View File

@ -11,10 +11,12 @@
return;
}
if ($config['smart_build']) {
$action = generation_strategy('sb_ukko', array());
if ($action == 'delete') {
file_unlink($settings['uri'] . '/index.html');
}
else {
elseif ($action == 'rebuild') {
file_write($settings['uri'] . '/index.html', $ukko->build());
}
}