advanced build (1/2): a small refactor of index generating procedure; generation strategies

inc/config.php

@@ -1203,16 +1203,74 @@
 	// Try not to build pages when we shouldn't have to.
 	$config['try_smarter'] = true;
 
-	// EXPERIMENTAL: Defer static HTML building to a moment, when a given file is actually accessed.
+/*
-	// Warning: This option won't run out of the box. You need to tell your webserver, that a file
+ * ====================
-	// for serving 403 and 404 pages is /smart_build.php. Also, you need to turn off indexes.
+ *  Advanced build
+ * ====================
+ */
+
+	// Strategies for file generation. Also known as an "advanced build". If you don't have performance
+	// issues, you can safely ignore that part, because it's hard to configure and won't even work on
+	// your free webhosting.
+	//
+	// A strategy is a function, that given the PHP environment and ($fun, $array) variable pair, returns
+	// an $action array or false.
+	//
+	// $fun - a controller function name, see inc/controller.php. This is named after functions, so that
+	//        we can generate the files in daemon.
+	//
+	// $array - arguments to be passed
+	//
+	// $action - action to be taken. It's an array, and the first element of it is one of the following:
+	//   * "immediate" - generate the page immediately
+	//   * "defer" - defer page generation to a moment a worker daemon gets to build it (serving a stale
+	//               page in the meantime). The remaining arguments are daemon-specific. Daemon isn't
+	//               implemented yet :DDDD inb4 while(true) { generate(Queue::Get()) }; (which is probably it).
+	//   * "build_on_load" - defer page generation to a moment, when the user actually accesses the page.
+	//                       This is a smart_build behaviour. You shouldn't use this one too much, if you
+	//                       use it for active boards, the server may choke due to a possible race condition.
+	//                       See my blog post: https://engine.vichan.net/blog/res/2.html
+	//
+	// So, let's assume we want to build a thread 1324 on board /b/, because a new post appeared there.
+	// We try the first strategy, giving it arguments: 'sb_thread', array('b', 1324). The strategy will
+	// now return a value $action, denoting an action to do. If $action is false, we try another strategy.
+	//
+	// As I said, configuration isn't easy.
+	//
+	// 1. chmod 0777 directories: tmp/locks/ and tmp/queue/.
+	// 2. serve 403 and 404 requests to go thru smart_build.php
+	//    for nginx, this blog post contains config snippets: https://engine.vichan.net/blog/res/2.html
+	// 3. disable indexes in your webserver
+	// 4. launch any number of daemons (eg. twice your number of threads?) using the command:
+	//    $ tools/worker.php
+	//    You don't need to do that step if you are not going to use the "defer" option.
+	// 5. enable smart_build_helper (see below)
+	// 6. edit the strategies (see inc/functions.php for the builtin ones). You can use lambdas. I will test
+	//    various ones and include one that works best for me.
+	$config['generation_strategies'] = array();
+	// Add a sane strategy. It forces to immediately generate a page user is about to land on. Otherwise,
+	// it has no opinion, so it needs a fallback strategy.
+	$config['generation_strategies'][] = 'strategy_sane';
+	// Add an immediate catch-all strategy. This is the default function of imageboards: generate all pages
+	// on post time.
+	$config['generation_strategies'][] = 'strategy_immediate';
+	// NOT RECOMMENDED: Instead of an all-"immediate" strategy, you can use an all-"build_on_load" one (used
+	// to be initialized using $config['smart_build']; ) for all pages instead of those to be build
+	// immediately. A rebuild done in this mode should remove all your static files
+	// $config['generation_strategies'][1] = 'strategy_smart_build';
+
+	// Deprecated. Leave it false. See above.
 	$config['smart_build'] = false;
 
-	// Smart build related: when a file doesn't exist, where should we redirect?
+	// Use smart_build.php for dispatching missing requests. It may be useful without smart_build or advanced
+	// build, for example it will regenerate the missing files.
+	$config['smart_build_helper'] = true;
+
+	// smart_build.php: when a file doesn't exist, where should we redirect?
 	$config['page_404'] = '/404.html';
 
-	// Smart build related: extra entrypoints.
+	// Extra controller entrypoints. Controller is used only by smart_build and advanced build.
-	$config['smart_build_entrypoints'] = array();
+	$config['controller_entrypoints'] = array();
 
 /*
  * ====================

inc/functions.php

@@ -1319,7 +1319,8 @@ function thread_find_page($thread) {
 	return floor(($config['threads_per_page'] + $index) / $config['threads_per_page']);
 }
 
-function index($page, $mod=false) {
+// $brief means that we won't need to generate anything yet
+function index($page, $mod=false, $brief = false) {
 	global $board, $config, $debug;
 
 	$body = '';
@@ -1350,6 +1351,7 @@ function index($page, $mod=false) {
 				unset($cached);
 			}
 		}
+
 		if (!isset($cached)) {
 			$posts = prepare(sprintf("SELECT * FROM ``posts_%s`` WHERE `thread` = :id ORDER BY `id` DESC LIMIT :limit", $board['uri']));
 			$posts->bindValue(':id', $th['id']);
@@ -1389,7 +1391,10 @@ function index($page, $mod=false) {
 		}
 		
 		$threads[] = $thread;
-		$body .= $thread->build(true);
+
+		if (!$brief) {
+			$body .= $thread->build(true);
+		}
 	}
 
 	if ($config['file_board']) {
@@ -1610,27 +1615,28 @@ function checkMute() {
 function buildIndex($global_api = "yes") {
 	global $board, $config, $build_pages;
 
-	if (!$config['smart_build']) {
+	$catalog_api_action = generation_strategy('sb_api', array($board['uri']));
-		$pages = getPages();
-		if (!$config['try_smarter'])
-			$antibot = create_antibot($board['uri']);
 
-		if ($config['api']['enabled']) {
+	$pages = null;
-			$api = new Api();
+	$antibot = null;
-			$catalog = array();
+
-		}
+	if ($config['api']['enabled']) {
+		$api = new Api();
+		$catalog = array();
 	}
 
 	for ($page = 1; $page <= $config['max_pages']; $page++) {
 		$filename = $board['dir'] . ($page == 1 ? $config['file_index'] : sprintf($config['file_page'], $page));
 		$jsonFilename = $board['dir'] . ($page - 1) . '.json'; // pages should start from 0
 
-		if ((!$config['api']['enabled'] || $global_api == "skip" || $config['smart_build']) && $config['try_smarter']
+		$wont_build_this_page = $config['try_smarter'] && isset($build_pages) && !empty($build_pages) && !in_array($page, $build_pages);
-			 && isset($build_pages) && !empty($build_pages) && !in_array($page, $build_pages) )
+
+		if ((!$config['api']['enabled'] || $global_api == "skip") && $wont_build_this_page)
 			continue;
 
-		if (!$config['smart_build']) {
+		$action = generation_strategy('sb_board', array($board['uri'], $page));
-			$content = index($page);
+		if ($action == 'rebuild' || $catalog_api_action == 'rebuild') {
+			$content = index($page, false, $wont_build_this_page);
 			if (!$content)
 				break;
 
@@ -1641,17 +1647,21 @@ function buildIndex($global_api = "yes") {
 				file_write($jsonFilename, $json);
 
 				$catalog[$page-1] = $threads;
-			}
 
-			if ($config['api']['enabled'] && $global_api != "skip" && $config['try_smarter'] && isset($build_pages)
+				if ($wont_build_this_page) continue;
-				&& !empty($build_pages) && !in_array($page, $build_pages) )
+			}
-				continue;
 
 			if ($config['try_smarter']) {
 				$antibot = create_antibot($board['uri'], 0 - $page);
 				$content['current_page'] = $page;
 			}
+			elseif (!$antibot) {
+				create_antibot($board['uri']);
+			}
 			$antibot->reset();
+			if (!$pages) {
+				$pages = getPages();
+			}
 			$content['pages'] = $pages;
 			$content['pages'][$page-1]['selected'] = true;
 			$content['btn'] = getPageButtons($content['pages']);
@@ -1659,13 +1669,14 @@ function buildIndex($global_api = "yes") {
 
 			file_write($filename, Element('index.html', $content));
 		}
-		else {
+		elseif ($action == 'delete' || $catalog_api_action == 'delete') {
 			file_unlink($filename);
 			file_unlink($jsonFilename);
 		}
 	}
 
-	if (!$config['smart_build'] && $page < $config['max_pages']) {
+	// $action is an action for our last page
+	if (($catalog_api_action == 'rebuild' || $action == 'rebuild' || $action == 'delete') && $page < $config['max_pages']) {
 		for (;$page<=$config['max_pages'];$page++) {
 			$filename = $board['dir'] . ($page==1 ? $config['file_index'] : sprintf($config['file_page'], $page));
 			file_unlink($filename);
@@ -1679,13 +1690,13 @@ function buildIndex($global_api = "yes") {
 
 	// json api catalog
 	if ($config['api']['enabled'] && $global_api != "skip") {
-		if ($config['smart_build']) {
+		if ($catalog_api_action == 'delete') {
 			$jsonFilename = $board['dir'] . 'catalog.json';
 			file_unlink($jsonFilename);
 			$jsonFilename = $board['dir'] . 'threads.json';
 			file_unlink($jsonFilename);
 		}
-		else {
+		elseif ($catalog_api_action == 'rebuild') {
 			$json = json_encode($api->translateCatalog($catalog));
 			$jsonFilename = $board['dir'] . 'catalog.json';
 			file_write($jsonFilename, $json);
@@ -2204,7 +2215,9 @@ function buildThread($id, $return = false, $mod = false) {
 	if ($config['try_smarter'] && !$mod)
 		$build_pages[] = thread_find_page($id);
 
-	if (!$config['smart_build'] || $return || $mod) {
+	$action = generation_strategy('sb_thread', array($board['uri'], $id));
+
+	if ($action == 'rebuild' || $return || $mod) {
 		$query = prepare(sprintf("SELECT * FROM ``posts_%s`` WHERE (`thread` IS NULL AND `id` = :id) OR `thread` = :id ORDER BY `thread`,`id`", $board['uri']));
 		$query->bindValue(':id', $id, PDO::PARAM_INT);
 		$query->execute() or error(db_error($query));
@@ -2239,26 +2252,26 @@ function buildThread($id, $return = false, $mod = false) {
 		));
 
 		// json api
-		if ($config['api']['enabled']) {
+		if ($config['api']['enabled'] && !$mod) {
 			$api = new Api();
 			$json = json_encode($api->translateThread($thread));
 			$jsonFilename = $board['dir'] . $config['dir']['res'] . $id . '.json';
 			file_write($jsonFilename, $json);
 		}
 	}
-	else {
+	elseif($action == 'delete') {
 		$jsonFilename = $board['dir'] . $config['dir']['res'] . $id . '.json';
 		file_unlink($jsonFilename);
 	}
 
-	if ($config['smart_build'] && !$return && !$mod) {
+	if ($action == 'delete' && !$return && !$mod) {
 		$noko50fn = $board['dir'] . $config['dir']['res'] . link_for(array('id' => $id), true);
 		file_unlink($noko50fn);
 
 		file_unlink($board['dir'] . $config['dir']['res'] . link_for(array('id' => $id)));
-	} else if ($return) {
+	} elseif ($return) {
 		return $body;
-	} else {
+	} elseif ($action == 'rebuild') {
 		$noko50fn = $board['dir'] . $config['dir']['res'] . link_for($thread, true);
 		if ($hasnoko50 || file_exists($noko50fn)) {
 			buildThread50($id, $return, $mod, $thread, $antibot);
@@ -2788,3 +2801,36 @@ function markdown($s) {
 
 	return $pd->text($s);
 }
+
+function generation_strategy($fun, $array=array()) { global $config;
+	$action = false;
+
+	foreach ($config['generation_strategies'] as $s) {
+		if ($strategy = $s($fun, $array)) {
+			break;
+		}
+	}
+
+	switch ($strategy[0]) {
+		case 'immediate':
+			return 'rebuild';
+		case 'defer':
+			// Ok, it gets interesting here :)
+			Queue::add(serialize(array('build', $fun, $array)));
+			return 'ignore';
+		case 'build_on_load':
+			return 'delete';
+	}
+}
+
+function strategy_immediate($fun, $array) {
+	return array('immediate');
+}
+
+function strategy_smart_build($fun, $array) {
+	return array('build_on_load');
+}
+
+function strategy_sane($fun, $array) { global $config;
+	return false;
+}

inc/route.php

@@ -7,7 +7,7 @@
 
 defined('TINYBOARD') or exit;
 
-function route($path) {
+function route($path) { global $config;
   $entrypoints = array();
 
   $entrypoints['/%b/']                       = 'sb_board';
@@ -33,8 +33,11 @@ function route($path) {
   $entrypoints['/*/index.html']    = 'sb_ukko';
   $entrypoints['/recent.html']     = 'sb_recent';
   $entrypoints['/%b/catalog.html'] = 'sb_catalog';
+  $entrypoints['/%b/index.rss']    = 'sb_catalog';
   $entrypoints['/sitemap.xml']     = 'sb_sitemap';
 
+  $entrypoints = array_merge($entrypoints, $config['controller_entrypoints']);
+
   $reached = false;
 
   list($request) = explode('?', $path);

smart_build.php

@@ -3,14 +3,16 @@ require_once("inc/functions.php");
 require_once("inc/route.php");
 require_once("inc/controller.php");
 
-if (!$config['smart_build'] && !$config["smart_build_helper"]) {
+if (!$config["smart_build_helper"]) {
-  die('You need to enable $config["smart_build"] or $config["smart_build_helper"]');
+  die('You need to enable $config["smart_build_helper"]');
 }
 
 $config['smart_build'] = false; // Let's disable it, so we can build the page for real
+$config['generation_strategies'] = array('strategy_immediate');
 
 function after_open_board() { global $config;
   $config['smart_build'] = false;
+  $config['generation_strategies'] = array('strategy_immediate');
 };
 
 $request = $_SERVER['REQUEST_URI'];
@@ -59,6 +61,9 @@ if ($reached) {
   elseif (preg_match('/\.xml$/', $request)) {
     header("Content-Type", "application/xml");
   }
+  elseif (preg_match('/\.rss$/', $request)) {
+    header("Content-Type", "application/rss+xml");
+  }
   else {
     header("Content-Type", "text/html; charset=utf-8");
   }

templates/themes/catalog/theme.php

@@ -16,20 +16,25 @@
 		if ($action == 'all') {
 			foreach ($boards as $board) {
 				$b = new Catalog();
-				if ($config['smart_build']) {
+
+				$action = generation_strategy("sb_catalog", array($board));
+				if ($action == 'delete') {
 					file_unlink($config['dir']['home'] . $board . '/catalog.html');
+					file_unlink($config['dir']['home'] . $board . '/index.rss');
 				}
-				else {
+				elseif ($action == 'rebuild') {
 					$b->build($settings, $board);
 				}
 			}
 		} elseif ($action == 'post-thread' || ($settings['update_on_posts'] && $action == 'post') || ($settings['update_on_posts'] && $action == 'post-delete') && in_array($board, $boards)) {
 			$b = new Catalog();
 
-			if ($config['smart_build']) {
+			$action = generation_strategy("sb_catalog", array($board));
+			if ($action == 'delete') {
 				file_unlink($config['dir']['home'] . $board . '/catalog.html');
+				file_unlink($config['dir']['home'] . $board . '/index.rss');
 			}
-			else {
+			elseif ($action == 'rebuild') {
 				$b->build($settings, $board);
 			}
 		}

templates/themes/recent/theme.php

@@ -25,10 +25,11 @@
 			$this->excluded = explode(' ', $settings['exclude']);
 			
 			if ($action == 'all' || $action == 'post' || $action == 'post-thread' || $action == 'post-delete') {
-				if ($config['smart_build']) {
+				$action = generation_strategy('sb_recent', array());
+				if ($action == 'delete') {
 					file_unlink($config['dir']['home'] . $settings['html']);
 				}
-				else {
+				elseif ($action == 'rebuild') {
 					file_write($config['dir']['home'] . $settings['html'], $this->homepage($settings));
 				}
 			}

templates/themes/sitemap/theme.php

@@ -23,10 +23,12 @@
 			}
 		}
 
-		if ($config['smart_build']) {
+		$action = generation_strategy('sb_sitemap', array());
+
+		if ($action == 'delete') {
 			file_unlink($settings['path']);
 		}
-		else {		
+		elseif ($action == 'rebuild') {
 			$boards = explode(' ', $settings['boards']);
 		
 			$threads = array();

templates/themes/ukko/theme.php

@@ -11,10 +11,12 @@
 			return;
 		}
 
-		if ($config['smart_build']) {
+		$action = generation_strategy('sb_ukko', array());
+
+		if ($action == 'delete') {
 			file_unlink($settings['uri'] . '/index.html');
 		}
-		else {
+		elseif ($action == 'rebuild') {
 			file_write($settings['uri'] . '/index.html', $ukko->build());
 		}
 	}