mirror of
https://github.com/vichan-devel/vichan.git
synced 2024-11-12 01:50:48 +01:00
advanced build (1/2): a small refactor of index generating procedure; generation strategies
This commit is contained in:
parent
644f227ab3
commit
b6f0317bde
@ -1203,16 +1203,74 @@
|
||||
// Try not to build pages when we shouldn't have to.
|
||||
$config['try_smarter'] = true;
|
||||
|
||||
// EXPERIMENTAL: Defer static HTML building to a moment, when a given file is actually accessed.
|
||||
// Warning: This option won't run out of the box. You need to tell your webserver, that a file
|
||||
// for serving 403 and 404 pages is /smart_build.php. Also, you need to turn off indexes.
|
||||
/*
|
||||
* ====================
|
||||
* Advanced build
|
||||
* ====================
|
||||
*/
|
||||
|
||||
// Strategies for file generation. Also known as an "advanced build". If you don't have performance
|
||||
// issues, you can safely ignore that part, because it's hard to configure and won't even work on
|
||||
// your free webhosting.
|
||||
//
|
||||
// A strategy is a function, that given the PHP environment and ($fun, $array) variable pair, returns
|
||||
// an $action array or false.
|
||||
//
|
||||
// $fun - a controller function name, see inc/controller.php. This is named after functions, so that
|
||||
// we can generate the files in daemon.
|
||||
//
|
||||
// $array - arguments to be passed
|
||||
//
|
||||
// $action - action to be taken. It's an array, and the first element of it is one of the following:
|
||||
// * "immediate" - generate the page immediately
|
||||
// * "defer" - defer page generation to a moment a worker daemon gets to build it (serving a stale
|
||||
// page in the meantime). The remaining arguments are daemon-specific. Daemon isn't
|
||||
// implemented yet :DDDD inb4 while(true) { generate(Queue::Get()) }; (which is probably it).
|
||||
// * "build_on_load" - defer page generation to a moment, when the user actually accesses the page.
|
||||
// This is a smart_build behaviour. You shouldn't use this one too much, if you
|
||||
// use it for active boards, the server may choke due to a possible race condition.
|
||||
// See my blog post: https://engine.vichan.net/blog/res/2.html
|
||||
//
|
||||
// So, let's assume we want to build a thread 1324 on board /b/, because a new post appeared there.
|
||||
// We try the first strategy, giving it arguments: 'sb_thread', array('b', 1324). The strategy will
|
||||
// now return a value $action, denoting an action to do. If $action is false, we try another strategy.
|
||||
//
|
||||
// As I said, configuration isn't easy.
|
||||
//
|
||||
// 1. chmod 0777 directories: tmp/locks/ and tmp/queue/.
|
||||
// 2. serve 403 and 404 requests to go thru smart_build.php
|
||||
// for nginx, this blog post contains config snippets: https://engine.vichan.net/blog/res/2.html
|
||||
// 3. disable indexes in your webserver
|
||||
// 4. launch any number of daemons (eg. twice your number of threads?) using the command:
|
||||
// $ tools/worker.php
|
||||
// You don't need to do that step if you are not going to use the "defer" option.
|
||||
// 5. enable smart_build_helper (see below)
|
||||
// 6. edit the strategies (see inc/functions.php for the builtin ones). You can use lambdas. I will test
|
||||
// various ones and include one that works best for me.
|
||||
$config['generation_strategies'] = array();
|
||||
// Add a sane strategy. It forces to immediately generate a page user is about to land on. Otherwise,
|
||||
// it has no opinion, so it needs a fallback strategy.
|
||||
$config['generation_strategies'][] = 'strategy_sane';
|
||||
// Add an immediate catch-all strategy. This is the default function of imageboards: generate all pages
|
||||
// on post time.
|
||||
$config['generation_strategies'][] = 'strategy_immediate';
|
||||
// NOT RECOMMENDED: Instead of an all-"immediate" strategy, you can use an all-"build_on_load" one (used
|
||||
// to be initialized using $config['smart_build']; ) for all pages instead of those to be build
|
||||
// immediately. A rebuild done in this mode should remove all your static files
|
||||
// $config['generation_strategies'][1] = 'strategy_smart_build';
|
||||
|
||||
// Deprecated. Leave it false. See above.
|
||||
$config['smart_build'] = false;
|
||||
|
||||
// Smart build related: when a file doesn't exist, where should we redirect?
|
||||
// Use smart_build.php for dispatching missing requests. It may be useful without smart_build or advanced
|
||||
// build, for example it will regenerate the missing files.
|
||||
$config['smart_build_helper'] = true;
|
||||
|
||||
// smart_build.php: when a file doesn't exist, where should we redirect?
|
||||
$config['page_404'] = '/404.html';
|
||||
|
||||
// Smart build related: extra entrypoints.
|
||||
$config['smart_build_entrypoints'] = array();
|
||||
// Extra controller entrypoints. Controller is used only by smart_build and advanced build.
|
||||
$config['controller_entrypoints'] = array();
|
||||
|
||||
/*
|
||||
* ====================
|
||||
|
@ -1319,7 +1319,8 @@ function thread_find_page($thread) {
|
||||
return floor(($config['threads_per_page'] + $index) / $config['threads_per_page']);
|
||||
}
|
||||
|
||||
function index($page, $mod=false) {
|
||||
// $brief means that we won't need to generate anything yet
|
||||
function index($page, $mod=false, $brief = false) {
|
||||
global $board, $config, $debug;
|
||||
|
||||
$body = '';
|
||||
@ -1350,6 +1351,7 @@ function index($page, $mod=false) {
|
||||
unset($cached);
|
||||
}
|
||||
}
|
||||
|
||||
if (!isset($cached)) {
|
||||
$posts = prepare(sprintf("SELECT * FROM ``posts_%s`` WHERE `thread` = :id ORDER BY `id` DESC LIMIT :limit", $board['uri']));
|
||||
$posts->bindValue(':id', $th['id']);
|
||||
@ -1389,7 +1391,10 @@ function index($page, $mod=false) {
|
||||
}
|
||||
|
||||
$threads[] = $thread;
|
||||
$body .= $thread->build(true);
|
||||
|
||||
if (!$brief) {
|
||||
$body .= $thread->build(true);
|
||||
}
|
||||
}
|
||||
|
||||
if ($config['file_board']) {
|
||||
@ -1610,27 +1615,28 @@ function checkMute() {
|
||||
function buildIndex($global_api = "yes") {
|
||||
global $board, $config, $build_pages;
|
||||
|
||||
if (!$config['smart_build']) {
|
||||
$pages = getPages();
|
||||
if (!$config['try_smarter'])
|
||||
$antibot = create_antibot($board['uri']);
|
||||
$catalog_api_action = generation_strategy('sb_api', array($board['uri']));
|
||||
|
||||
if ($config['api']['enabled']) {
|
||||
$api = new Api();
|
||||
$catalog = array();
|
||||
}
|
||||
$pages = null;
|
||||
$antibot = null;
|
||||
|
||||
if ($config['api']['enabled']) {
|
||||
$api = new Api();
|
||||
$catalog = array();
|
||||
}
|
||||
|
||||
for ($page = 1; $page <= $config['max_pages']; $page++) {
|
||||
$filename = $board['dir'] . ($page == 1 ? $config['file_index'] : sprintf($config['file_page'], $page));
|
||||
$jsonFilename = $board['dir'] . ($page - 1) . '.json'; // pages should start from 0
|
||||
|
||||
if ((!$config['api']['enabled'] || $global_api == "skip" || $config['smart_build']) && $config['try_smarter']
|
||||
&& isset($build_pages) && !empty($build_pages) && !in_array($page, $build_pages) )
|
||||
$wont_build_this_page = $config['try_smarter'] && isset($build_pages) && !empty($build_pages) && !in_array($page, $build_pages);
|
||||
|
||||
if ((!$config['api']['enabled'] || $global_api == "skip") && $wont_build_this_page)
|
||||
continue;
|
||||
|
||||
if (!$config['smart_build']) {
|
||||
$content = index($page);
|
||||
$action = generation_strategy('sb_board', array($board['uri'], $page));
|
||||
if ($action == 'rebuild' || $catalog_api_action == 'rebuild') {
|
||||
$content = index($page, false, $wont_build_this_page);
|
||||
if (!$content)
|
||||
break;
|
||||
|
||||
@ -1641,17 +1647,21 @@ function buildIndex($global_api = "yes") {
|
||||
file_write($jsonFilename, $json);
|
||||
|
||||
$catalog[$page-1] = $threads;
|
||||
}
|
||||
|
||||
if ($config['api']['enabled'] && $global_api != "skip" && $config['try_smarter'] && isset($build_pages)
|
||||
&& !empty($build_pages) && !in_array($page, $build_pages) )
|
||||
continue;
|
||||
if ($wont_build_this_page) continue;
|
||||
}
|
||||
|
||||
if ($config['try_smarter']) {
|
||||
$antibot = create_antibot($board['uri'], 0 - $page);
|
||||
$content['current_page'] = $page;
|
||||
}
|
||||
elseif (!$antibot) {
|
||||
create_antibot($board['uri']);
|
||||
}
|
||||
$antibot->reset();
|
||||
if (!$pages) {
|
||||
$pages = getPages();
|
||||
}
|
||||
$content['pages'] = $pages;
|
||||
$content['pages'][$page-1]['selected'] = true;
|
||||
$content['btn'] = getPageButtons($content['pages']);
|
||||
@ -1659,13 +1669,14 @@ function buildIndex($global_api = "yes") {
|
||||
|
||||
file_write($filename, Element('index.html', $content));
|
||||
}
|
||||
else {
|
||||
elseif ($action == 'delete' || $catalog_api_action == 'delete') {
|
||||
file_unlink($filename);
|
||||
file_unlink($jsonFilename);
|
||||
}
|
||||
}
|
||||
|
||||
if (!$config['smart_build'] && $page < $config['max_pages']) {
|
||||
// $action is an action for our last page
|
||||
if (($catalog_api_action == 'rebuild' || $action == 'rebuild' || $action == 'delete') && $page < $config['max_pages']) {
|
||||
for (;$page<=$config['max_pages'];$page++) {
|
||||
$filename = $board['dir'] . ($page==1 ? $config['file_index'] : sprintf($config['file_page'], $page));
|
||||
file_unlink($filename);
|
||||
@ -1679,13 +1690,13 @@ function buildIndex($global_api = "yes") {
|
||||
|
||||
// json api catalog
|
||||
if ($config['api']['enabled'] && $global_api != "skip") {
|
||||
if ($config['smart_build']) {
|
||||
if ($catalog_api_action == 'delete') {
|
||||
$jsonFilename = $board['dir'] . 'catalog.json';
|
||||
file_unlink($jsonFilename);
|
||||
$jsonFilename = $board['dir'] . 'threads.json';
|
||||
file_unlink($jsonFilename);
|
||||
}
|
||||
else {
|
||||
elseif ($catalog_api_action == 'rebuild') {
|
||||
$json = json_encode($api->translateCatalog($catalog));
|
||||
$jsonFilename = $board['dir'] . 'catalog.json';
|
||||
file_write($jsonFilename, $json);
|
||||
@ -2204,7 +2215,9 @@ function buildThread($id, $return = false, $mod = false) {
|
||||
if ($config['try_smarter'] && !$mod)
|
||||
$build_pages[] = thread_find_page($id);
|
||||
|
||||
if (!$config['smart_build'] || $return || $mod) {
|
||||
$action = generation_strategy('sb_thread', array($board['uri'], $id));
|
||||
|
||||
if ($action == 'rebuild' || $return || $mod) {
|
||||
$query = prepare(sprintf("SELECT * FROM ``posts_%s`` WHERE (`thread` IS NULL AND `id` = :id) OR `thread` = :id ORDER BY `thread`,`id`", $board['uri']));
|
||||
$query->bindValue(':id', $id, PDO::PARAM_INT);
|
||||
$query->execute() or error(db_error($query));
|
||||
@ -2239,26 +2252,26 @@ function buildThread($id, $return = false, $mod = false) {
|
||||
));
|
||||
|
||||
// json api
|
||||
if ($config['api']['enabled']) {
|
||||
if ($config['api']['enabled'] && !$mod) {
|
||||
$api = new Api();
|
||||
$json = json_encode($api->translateThread($thread));
|
||||
$jsonFilename = $board['dir'] . $config['dir']['res'] . $id . '.json';
|
||||
file_write($jsonFilename, $json);
|
||||
}
|
||||
}
|
||||
else {
|
||||
elseif($action == 'delete') {
|
||||
$jsonFilename = $board['dir'] . $config['dir']['res'] . $id . '.json';
|
||||
file_unlink($jsonFilename);
|
||||
}
|
||||
|
||||
if ($config['smart_build'] && !$return && !$mod) {
|
||||
if ($action == 'delete' && !$return && !$mod) {
|
||||
$noko50fn = $board['dir'] . $config['dir']['res'] . link_for(array('id' => $id), true);
|
||||
file_unlink($noko50fn);
|
||||
|
||||
file_unlink($board['dir'] . $config['dir']['res'] . link_for(array('id' => $id)));
|
||||
} else if ($return) {
|
||||
} elseif ($return) {
|
||||
return $body;
|
||||
} else {
|
||||
} elseif ($action == 'rebuild') {
|
||||
$noko50fn = $board['dir'] . $config['dir']['res'] . link_for($thread, true);
|
||||
if ($hasnoko50 || file_exists($noko50fn)) {
|
||||
buildThread50($id, $return, $mod, $thread, $antibot);
|
||||
@ -2788,3 +2801,36 @@ function markdown($s) {
|
||||
|
||||
return $pd->text($s);
|
||||
}
|
||||
|
||||
function generation_strategy($fun, $array=array()) { global $config;
|
||||
$action = false;
|
||||
|
||||
foreach ($config['generation_strategies'] as $s) {
|
||||
if ($strategy = $s($fun, $array)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
switch ($strategy[0]) {
|
||||
case 'immediate':
|
||||
return 'rebuild';
|
||||
case 'defer':
|
||||
// Ok, it gets interesting here :)
|
||||
Queue::add(serialize(array('build', $fun, $array)));
|
||||
return 'ignore';
|
||||
case 'build_on_load':
|
||||
return 'delete';
|
||||
}
|
||||
}
|
||||
|
||||
function strategy_immediate($fun, $array) {
|
||||
return array('immediate');
|
||||
}
|
||||
|
||||
function strategy_smart_build($fun, $array) {
|
||||
return array('build_on_load');
|
||||
}
|
||||
|
||||
function strategy_sane($fun, $array) { global $config;
|
||||
return false;
|
||||
}
|
||||
|
@ -7,7 +7,7 @@
|
||||
|
||||
defined('TINYBOARD') or exit;
|
||||
|
||||
function route($path) {
|
||||
function route($path) { global $config;
|
||||
$entrypoints = array();
|
||||
|
||||
$entrypoints['/%b/'] = 'sb_board';
|
||||
@ -33,8 +33,11 @@ function route($path) {
|
||||
$entrypoints['/*/index.html'] = 'sb_ukko';
|
||||
$entrypoints['/recent.html'] = 'sb_recent';
|
||||
$entrypoints['/%b/catalog.html'] = 'sb_catalog';
|
||||
$entrypoints['/%b/index.rss'] = 'sb_catalog';
|
||||
$entrypoints['/sitemap.xml'] = 'sb_sitemap';
|
||||
|
||||
$entrypoints = array_merge($entrypoints, $config['controller_entrypoints']);
|
||||
|
||||
$reached = false;
|
||||
|
||||
list($request) = explode('?', $path);
|
||||
|
@ -3,14 +3,16 @@ require_once("inc/functions.php");
|
||||
require_once("inc/route.php");
|
||||
require_once("inc/controller.php");
|
||||
|
||||
if (!$config['smart_build'] && !$config["smart_build_helper"]) {
|
||||
die('You need to enable $config["smart_build"] or $config["smart_build_helper"]');
|
||||
if (!$config["smart_build_helper"]) {
|
||||
die('You need to enable $config["smart_build_helper"]');
|
||||
}
|
||||
|
||||
$config['smart_build'] = false; // Let's disable it, so we can build the page for real
|
||||
$config['generation_strategies'] = array('strategy_immediate');
|
||||
|
||||
function after_open_board() { global $config;
|
||||
$config['smart_build'] = false;
|
||||
$config['generation_strategies'] = array('strategy_immediate');
|
||||
};
|
||||
|
||||
$request = $_SERVER['REQUEST_URI'];
|
||||
@ -59,6 +61,9 @@ if ($reached) {
|
||||
elseif (preg_match('/\.xml$/', $request)) {
|
||||
header("Content-Type", "application/xml");
|
||||
}
|
||||
elseif (preg_match('/\.rss$/', $request)) {
|
||||
header("Content-Type", "application/rss+xml");
|
||||
}
|
||||
else {
|
||||
header("Content-Type", "text/html; charset=utf-8");
|
||||
}
|
||||
|
@ -16,20 +16,25 @@
|
||||
if ($action == 'all') {
|
||||
foreach ($boards as $board) {
|
||||
$b = new Catalog();
|
||||
if ($config['smart_build']) {
|
||||
|
||||
$action = generation_strategy("sb_catalog", array($board));
|
||||
if ($action == 'delete') {
|
||||
file_unlink($config['dir']['home'] . $board . '/catalog.html');
|
||||
file_unlink($config['dir']['home'] . $board . '/index.rss');
|
||||
}
|
||||
else {
|
||||
elseif ($action == 'rebuild') {
|
||||
$b->build($settings, $board);
|
||||
}
|
||||
}
|
||||
} elseif ($action == 'post-thread' || ($settings['update_on_posts'] && $action == 'post') || ($settings['update_on_posts'] && $action == 'post-delete') && in_array($board, $boards)) {
|
||||
$b = new Catalog();
|
||||
|
||||
if ($config['smart_build']) {
|
||||
$action = generation_strategy("sb_catalog", array($board));
|
||||
if ($action == 'delete') {
|
||||
file_unlink($config['dir']['home'] . $board . '/catalog.html');
|
||||
file_unlink($config['dir']['home'] . $board . '/index.rss');
|
||||
}
|
||||
else {
|
||||
elseif ($action == 'rebuild') {
|
||||
$b->build($settings, $board);
|
||||
}
|
||||
}
|
||||
|
@ -25,10 +25,11 @@
|
||||
$this->excluded = explode(' ', $settings['exclude']);
|
||||
|
||||
if ($action == 'all' || $action == 'post' || $action == 'post-thread' || $action == 'post-delete') {
|
||||
if ($config['smart_build']) {
|
||||
$action = generation_strategy('sb_recent', array());
|
||||
if ($action == 'delete') {
|
||||
file_unlink($config['dir']['home'] . $settings['html']);
|
||||
}
|
||||
else {
|
||||
elseif ($action == 'rebuild') {
|
||||
file_write($config['dir']['home'] . $settings['html'], $this->homepage($settings));
|
||||
}
|
||||
}
|
||||
|
@ -23,10 +23,12 @@
|
||||
}
|
||||
}
|
||||
|
||||
if ($config['smart_build']) {
|
||||
$action = generation_strategy('sb_sitemap', array());
|
||||
|
||||
if ($action == 'delete') {
|
||||
file_unlink($settings['path']);
|
||||
}
|
||||
else {
|
||||
elseif ($action == 'rebuild') {
|
||||
$boards = explode(' ', $settings['boards']);
|
||||
|
||||
$threads = array();
|
||||
|
@ -11,10 +11,12 @@
|
||||
return;
|
||||
}
|
||||
|
||||
if ($config['smart_build']) {
|
||||
$action = generation_strategy('sb_ukko', array());
|
||||
|
||||
if ($action == 'delete') {
|
||||
file_unlink($settings['uri'] . '/index.html');
|
||||
}
|
||||
else {
|
||||
elseif ($action == 'rebuild') {
|
||||
file_write($settings['uri'] . '/index.html', $ukko->build());
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user