From b6f0317bde1b6d9a9ac5d81a660865a7aa9ad13d Mon Sep 17 00:00:00 2001 From: czaks Date: Sun, 8 May 2016 10:54:30 +0200 Subject: [PATCH] advanced build (1/2): a small refactor of index generating procedure; generation strategies --- inc/config.php | 70 ++++++++++++++++++-- inc/functions.php | 102 +++++++++++++++++++++-------- inc/route.php | 5 +- smart_build.php | 9 ++- templates/themes/catalog/theme.php | 13 ++-- templates/themes/recent/theme.php | 5 +- templates/themes/sitemap/theme.php | 6 +- templates/themes/ukko/theme.php | 6 +- 8 files changed, 169 insertions(+), 47 deletions(-) diff --git a/inc/config.php b/inc/config.php index 82fdc882..91804429 100644 --- a/inc/config.php +++ b/inc/config.php @@ -1203,16 +1203,74 @@ // Try not to build pages when we shouldn't have to. $config['try_smarter'] = true; - // EXPERIMENTAL: Defer static HTML building to a moment, when a given file is actually accessed. - // Warning: This option won't run out of the box. You need to tell your webserver, that a file - // for serving 403 and 404 pages is /smart_build.php. Also, you need to turn off indexes. +/* + * ==================== + * Advanced build + * ==================== + */ + + // Strategies for file generation. Also known as an "advanced build". If you don't have performance + // issues, you can safely ignore that part, because it's hard to configure and won't even work on + // your free webhosting. + // + // A strategy is a function, that given the PHP environment and ($fun, $array) variable pair, returns + // an $action array or false. + // + // $fun - a controller function name, see inc/controller.php. This is named after functions, so that + // we can generate the files in daemon. + // + // $array - arguments to be passed + // + // $action - action to be taken. It's an array, and the first element of it is one of the following: + // * "immediate" - generate the page immediately + // * "defer" - defer page generation to a moment a worker daemon gets to build it (serving a stale + // page in the meantime). The remaining arguments are daemon-specific. Daemon isn't + // implemented yet :DDDD inb4 while(true) { generate(Queue::Get()) }; (which is probably it). + // * "build_on_load" - defer page generation to a moment, when the user actually accesses the page. + // This is a smart_build behaviour. You shouldn't use this one too much, if you + // use it for active boards, the server may choke due to a possible race condition. + // See my blog post: https://engine.vichan.net/blog/res/2.html + // + // So, let's assume we want to build a thread 1324 on board /b/, because a new post appeared there. + // We try the first strategy, giving it arguments: 'sb_thread', array('b', 1324). The strategy will + // now return a value $action, denoting an action to do. If $action is false, we try another strategy. + // + // As I said, configuration isn't easy. + // + // 1. chmod 0777 directories: tmp/locks/ and tmp/queue/. + // 2. serve 403 and 404 requests to go thru smart_build.php + // for nginx, this blog post contains config snippets: https://engine.vichan.net/blog/res/2.html + // 3. disable indexes in your webserver + // 4. launch any number of daemons (eg. twice your number of threads?) using the command: + // $ tools/worker.php + // You don't need to do that step if you are not going to use the "defer" option. + // 5. enable smart_build_helper (see below) + // 6. edit the strategies (see inc/functions.php for the builtin ones). You can use lambdas. I will test + // various ones and include one that works best for me. + $config['generation_strategies'] = array(); + // Add a sane strategy. It forces to immediately generate a page user is about to land on. Otherwise, + // it has no opinion, so it needs a fallback strategy. + $config['generation_strategies'][] = 'strategy_sane'; + // Add an immediate catch-all strategy. This is the default function of imageboards: generate all pages + // on post time. + $config['generation_strategies'][] = 'strategy_immediate'; + // NOT RECOMMENDED: Instead of an all-"immediate" strategy, you can use an all-"build_on_load" one (used + // to be initialized using $config['smart_build']; ) for all pages instead of those to be build + // immediately. A rebuild done in this mode should remove all your static files + // $config['generation_strategies'][1] = 'strategy_smart_build'; + + // Deprecated. Leave it false. See above. $config['smart_build'] = false; - // Smart build related: when a file doesn't exist, where should we redirect? + // Use smart_build.php for dispatching missing requests. It may be useful without smart_build or advanced + // build, for example it will regenerate the missing files. + $config['smart_build_helper'] = true; + + // smart_build.php: when a file doesn't exist, where should we redirect? $config['page_404'] = '/404.html'; - // Smart build related: extra entrypoints. - $config['smart_build_entrypoints'] = array(); + // Extra controller entrypoints. Controller is used only by smart_build and advanced build. + $config['controller_entrypoints'] = array(); /* * ==================== diff --git a/inc/functions.php b/inc/functions.php index 1bd1f32f..2957c680 100755 --- a/inc/functions.php +++ b/inc/functions.php @@ -1319,7 +1319,8 @@ function thread_find_page($thread) { return floor(($config['threads_per_page'] + $index) / $config['threads_per_page']); } -function index($page, $mod=false) { +// $brief means that we won't need to generate anything yet +function index($page, $mod=false, $brief = false) { global $board, $config, $debug; $body = ''; @@ -1350,6 +1351,7 @@ function index($page, $mod=false) { unset($cached); } } + if (!isset($cached)) { $posts = prepare(sprintf("SELECT * FROM ``posts_%s`` WHERE `thread` = :id ORDER BY `id` DESC LIMIT :limit", $board['uri'])); $posts->bindValue(':id', $th['id']); @@ -1389,7 +1391,10 @@ function index($page, $mod=false) { } $threads[] = $thread; - $body .= $thread->build(true); + + if (!$brief) { + $body .= $thread->build(true); + } } if ($config['file_board']) { @@ -1610,27 +1615,28 @@ function checkMute() { function buildIndex($global_api = "yes") { global $board, $config, $build_pages; - if (!$config['smart_build']) { - $pages = getPages(); - if (!$config['try_smarter']) - $antibot = create_antibot($board['uri']); + $catalog_api_action = generation_strategy('sb_api', array($board['uri'])); - if ($config['api']['enabled']) { - $api = new Api(); - $catalog = array(); - } + $pages = null; + $antibot = null; + + if ($config['api']['enabled']) { + $api = new Api(); + $catalog = array(); } for ($page = 1; $page <= $config['max_pages']; $page++) { $filename = $board['dir'] . ($page == 1 ? $config['file_index'] : sprintf($config['file_page'], $page)); $jsonFilename = $board['dir'] . ($page - 1) . '.json'; // pages should start from 0 - if ((!$config['api']['enabled'] || $global_api == "skip" || $config['smart_build']) && $config['try_smarter'] - && isset($build_pages) && !empty($build_pages) && !in_array($page, $build_pages) ) + $wont_build_this_page = $config['try_smarter'] && isset($build_pages) && !empty($build_pages) && !in_array($page, $build_pages); + + if ((!$config['api']['enabled'] || $global_api == "skip") && $wont_build_this_page) continue; - if (!$config['smart_build']) { - $content = index($page); + $action = generation_strategy('sb_board', array($board['uri'], $page)); + if ($action == 'rebuild' || $catalog_api_action == 'rebuild') { + $content = index($page, false, $wont_build_this_page); if (!$content) break; @@ -1641,17 +1647,21 @@ function buildIndex($global_api = "yes") { file_write($jsonFilename, $json); $catalog[$page-1] = $threads; - } - if ($config['api']['enabled'] && $global_api != "skip" && $config['try_smarter'] && isset($build_pages) - && !empty($build_pages) && !in_array($page, $build_pages) ) - continue; + if ($wont_build_this_page) continue; + } if ($config['try_smarter']) { $antibot = create_antibot($board['uri'], 0 - $page); $content['current_page'] = $page; } + elseif (!$antibot) { + create_antibot($board['uri']); + } $antibot->reset(); + if (!$pages) { + $pages = getPages(); + } $content['pages'] = $pages; $content['pages'][$page-1]['selected'] = true; $content['btn'] = getPageButtons($content['pages']); @@ -1659,13 +1669,14 @@ function buildIndex($global_api = "yes") { file_write($filename, Element('index.html', $content)); } - else { + elseif ($action == 'delete' || $catalog_api_action == 'delete') { file_unlink($filename); file_unlink($jsonFilename); } } - if (!$config['smart_build'] && $page < $config['max_pages']) { + // $action is an action for our last page + if (($catalog_api_action == 'rebuild' || $action == 'rebuild' || $action == 'delete') && $page < $config['max_pages']) { for (;$page<=$config['max_pages'];$page++) { $filename = $board['dir'] . ($page==1 ? $config['file_index'] : sprintf($config['file_page'], $page)); file_unlink($filename); @@ -1679,13 +1690,13 @@ function buildIndex($global_api = "yes") { // json api catalog if ($config['api']['enabled'] && $global_api != "skip") { - if ($config['smart_build']) { + if ($catalog_api_action == 'delete') { $jsonFilename = $board['dir'] . 'catalog.json'; file_unlink($jsonFilename); $jsonFilename = $board['dir'] . 'threads.json'; file_unlink($jsonFilename); } - else { + elseif ($catalog_api_action == 'rebuild') { $json = json_encode($api->translateCatalog($catalog)); $jsonFilename = $board['dir'] . 'catalog.json'; file_write($jsonFilename, $json); @@ -2204,7 +2215,9 @@ function buildThread($id, $return = false, $mod = false) { if ($config['try_smarter'] && !$mod) $build_pages[] = thread_find_page($id); - if (!$config['smart_build'] || $return || $mod) { + $action = generation_strategy('sb_thread', array($board['uri'], $id)); + + if ($action == 'rebuild' || $return || $mod) { $query = prepare(sprintf("SELECT * FROM ``posts_%s`` WHERE (`thread` IS NULL AND `id` = :id) OR `thread` = :id ORDER BY `thread`,`id`", $board['uri'])); $query->bindValue(':id', $id, PDO::PARAM_INT); $query->execute() or error(db_error($query)); @@ -2239,26 +2252,26 @@ function buildThread($id, $return = false, $mod = false) { )); // json api - if ($config['api']['enabled']) { + if ($config['api']['enabled'] && !$mod) { $api = new Api(); $json = json_encode($api->translateThread($thread)); $jsonFilename = $board['dir'] . $config['dir']['res'] . $id . '.json'; file_write($jsonFilename, $json); } } - else { + elseif($action == 'delete') { $jsonFilename = $board['dir'] . $config['dir']['res'] . $id . '.json'; file_unlink($jsonFilename); } - if ($config['smart_build'] && !$return && !$mod) { + if ($action == 'delete' && !$return && !$mod) { $noko50fn = $board['dir'] . $config['dir']['res'] . link_for(array('id' => $id), true); file_unlink($noko50fn); file_unlink($board['dir'] . $config['dir']['res'] . link_for(array('id' => $id))); - } else if ($return) { + } elseif ($return) { return $body; - } else { + } elseif ($action == 'rebuild') { $noko50fn = $board['dir'] . $config['dir']['res'] . link_for($thread, true); if ($hasnoko50 || file_exists($noko50fn)) { buildThread50($id, $return, $mod, $thread, $antibot); @@ -2788,3 +2801,36 @@ function markdown($s) { return $pd->text($s); } + +function generation_strategy($fun, $array=array()) { global $config; + $action = false; + + foreach ($config['generation_strategies'] as $s) { + if ($strategy = $s($fun, $array)) { + break; + } + } + + switch ($strategy[0]) { + case 'immediate': + return 'rebuild'; + case 'defer': + // Ok, it gets interesting here :) + Queue::add(serialize(array('build', $fun, $array))); + return 'ignore'; + case 'build_on_load': + return 'delete'; + } +} + +function strategy_immediate($fun, $array) { + return array('immediate'); +} + +function strategy_smart_build($fun, $array) { + return array('build_on_load'); +} + +function strategy_sane($fun, $array) { global $config; + return false; +} diff --git a/inc/route.php b/inc/route.php index 66602d77..2a5c1732 100644 --- a/inc/route.php +++ b/inc/route.php @@ -7,7 +7,7 @@ defined('TINYBOARD') or exit; -function route($path) { +function route($path) { global $config; $entrypoints = array(); $entrypoints['/%b/'] = 'sb_board'; @@ -33,8 +33,11 @@ function route($path) { $entrypoints['/*/index.html'] = 'sb_ukko'; $entrypoints['/recent.html'] = 'sb_recent'; $entrypoints['/%b/catalog.html'] = 'sb_catalog'; + $entrypoints['/%b/index.rss'] = 'sb_catalog'; $entrypoints['/sitemap.xml'] = 'sb_sitemap'; + $entrypoints = array_merge($entrypoints, $config['controller_entrypoints']); + $reached = false; list($request) = explode('?', $path); diff --git a/smart_build.php b/smart_build.php index 58596055..7ca5fcbf 100644 --- a/smart_build.php +++ b/smart_build.php @@ -3,14 +3,16 @@ require_once("inc/functions.php"); require_once("inc/route.php"); require_once("inc/controller.php"); -if (!$config['smart_build'] && !$config["smart_build_helper"]) { - die('You need to enable $config["smart_build"] or $config["smart_build_helper"]'); +if (!$config["smart_build_helper"]) { + die('You need to enable $config["smart_build_helper"]'); } $config['smart_build'] = false; // Let's disable it, so we can build the page for real +$config['generation_strategies'] = array('strategy_immediate'); function after_open_board() { global $config; $config['smart_build'] = false; + $config['generation_strategies'] = array('strategy_immediate'); }; $request = $_SERVER['REQUEST_URI']; @@ -59,6 +61,9 @@ if ($reached) { elseif (preg_match('/\.xml$/', $request)) { header("Content-Type", "application/xml"); } + elseif (preg_match('/\.rss$/', $request)) { + header("Content-Type", "application/rss+xml"); + } else { header("Content-Type", "text/html; charset=utf-8"); } diff --git a/templates/themes/catalog/theme.php b/templates/themes/catalog/theme.php index 239d4dff..4f512c03 100644 --- a/templates/themes/catalog/theme.php +++ b/templates/themes/catalog/theme.php @@ -16,20 +16,25 @@ if ($action == 'all') { foreach ($boards as $board) { $b = new Catalog(); - if ($config['smart_build']) { + + $action = generation_strategy("sb_catalog", array($board)); + if ($action == 'delete') { file_unlink($config['dir']['home'] . $board . '/catalog.html'); + file_unlink($config['dir']['home'] . $board . '/index.rss'); } - else { + elseif ($action == 'rebuild') { $b->build($settings, $board); } } } elseif ($action == 'post-thread' || ($settings['update_on_posts'] && $action == 'post') || ($settings['update_on_posts'] && $action == 'post-delete') && in_array($board, $boards)) { $b = new Catalog(); - if ($config['smart_build']) { + $action = generation_strategy("sb_catalog", array($board)); + if ($action == 'delete') { file_unlink($config['dir']['home'] . $board . '/catalog.html'); + file_unlink($config['dir']['home'] . $board . '/index.rss'); } - else { + elseif ($action == 'rebuild') { $b->build($settings, $board); } } diff --git a/templates/themes/recent/theme.php b/templates/themes/recent/theme.php index f44e2529..95921c35 100644 --- a/templates/themes/recent/theme.php +++ b/templates/themes/recent/theme.php @@ -25,10 +25,11 @@ $this->excluded = explode(' ', $settings['exclude']); if ($action == 'all' || $action == 'post' || $action == 'post-thread' || $action == 'post-delete') { - if ($config['smart_build']) { + $action = generation_strategy('sb_recent', array()); + if ($action == 'delete') { file_unlink($config['dir']['home'] . $settings['html']); } - else { + elseif ($action == 'rebuild') { file_write($config['dir']['home'] . $settings['html'], $this->homepage($settings)); } } diff --git a/templates/themes/sitemap/theme.php b/templates/themes/sitemap/theme.php index 52779d53..6bc035bb 100644 --- a/templates/themes/sitemap/theme.php +++ b/templates/themes/sitemap/theme.php @@ -23,10 +23,12 @@ } } - if ($config['smart_build']) { + $action = generation_strategy('sb_sitemap', array()); + + if ($action == 'delete') { file_unlink($settings['path']); } - else { + elseif ($action == 'rebuild') { $boards = explode(' ', $settings['boards']); $threads = array(); diff --git a/templates/themes/ukko/theme.php b/templates/themes/ukko/theme.php index d6fc303c..e572c467 100644 --- a/templates/themes/ukko/theme.php +++ b/templates/themes/ukko/theme.php @@ -11,10 +11,12 @@ return; } - if ($config['smart_build']) { + $action = generation_strategy('sb_ukko', array()); + + if ($action == 'delete') { file_unlink($settings['uri'] . '/index.html'); } - else { + elseif ($action == 'rebuild') { file_write($settings['uri'] . '/index.html', $ukko->build()); } }