From 9494e0185c8661ed18c373ce46cbf9a66cb4d008 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Mon, 2 Sep 2013 09:46:42 +1000 Subject: [PATCH 01/77] new index on bans table --- install.php | 4 +++- install.sql | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/install.php b/install.php index d60e5aab..1732aa94 100644 --- a/install.php +++ b/install.php @@ -1,7 +1,7 @@ Date: Mon, 2 Sep 2013 11:41:17 +1000 Subject: [PATCH 02/77] another index change --- install.php | 6 +++++- install.sql | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/install.php b/install.php index 1732aa94..e8bdda8b 100644 --- a/install.php +++ b/install.php @@ -1,7 +1,7 @@ Date: Sun, 1 Sep 2013 11:20:57 -0400 Subject: [PATCH 03/77] Fixed working on some broken shared hostings. Thanks for Belarussian anon for reporting. --- inc/mod/pages.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inc/mod/pages.php b/inc/mod/pages.php index 2997ba6b..99bfcf9d 100644 --- a/inc/mod/pages.php +++ b/inc/mod/pages.php @@ -1446,7 +1446,7 @@ function mod_deletebyip($boardName, $post, $global = false) { if ($query->rowCount() < 1) error($config['error']['invalidpost']); - set_time_limit($config['mod']['rebuild_timelimit']); + @set_time_limit($config['mod']['rebuild_timelimit']); $threads_to_rebuild = array(); $threads_deleted = array(); @@ -1813,7 +1813,7 @@ function mod_rebuild() { error($config['error']['noaccess']); if (isset($_POST['rebuild'])) { - set_time_limit($config['mod']['rebuild_timelimit']); + @set_time_limit($config['mod']['rebuild_timelimit']); $log = array(); $boards = listBoards(); From 7f0de936082fb35d4e18fbadfe9849acdfc03ab6 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Fri, 6 Sep 2013 20:12:04 +1000 Subject: [PATCH 04/77] Cleaner check to make sure inc/ files aren't accessed directly. --- inc/anti-bot.php | 5 +---- inc/api.php | 1 + inc/cache.php | 5 +---- inc/database.php | 5 +---- inc/events.php | 5 +---- inc/filters.php | 5 +---- inc/functions.php | 2 ++ inc/image.php | 5 +---- inc/mod/auth.php | 5 +---- inc/mod/pages.php | 5 +---- inc/remote.php | 5 +---- inc/template.php | 5 +---- 12 files changed, 13 insertions(+), 40 deletions(-) diff --git a/inc/anti-bot.php b/inc/anti-bot.php index 41e0a1d6..dbe87e2b 100644 --- a/inc/anti-bot.php +++ b/inc/anti-bot.php @@ -4,10 +4,7 @@ * Copyright (c) 2010-2013 Tinyboard Development Group */ -if (realpath($_SERVER['SCRIPT_FILENAME']) == str_replace('\\', '/', __FILE__)) { - // You cannot request this file directly. - exit; -} +defined('TINYBOARD') or exit; $hidden_inputs_twig = array(); diff --git a/inc/api.php b/inc/api.php index a8331590..74043912 100644 --- a/inc/api.php +++ b/inc/api.php @@ -3,6 +3,7 @@ * Copyright (c) 2010-2013 Tinyboard Development Group */ +defined('TINYBOARD') or exit; /** * Class for generating json API compatible with 4chan API diff --git a/inc/cache.php b/inc/cache.php index 2f3c746c..95b1281f 100644 --- a/inc/cache.php +++ b/inc/cache.php @@ -4,10 +4,7 @@ * Copyright (c) 2010-2013 Tinyboard Development Group */ -if (realpath($_SERVER['SCRIPT_FILENAME']) == str_replace('\\', '/', __FILE__)) { - // You cannot request this file directly. - exit; -} +defined('TINYBOARD') or exit; class Cache { private static $cache; diff --git a/inc/database.php b/inc/database.php index 55e1ff81..d2f1af04 100644 --- a/inc/database.php +++ b/inc/database.php @@ -4,10 +4,7 @@ * Copyright (c) 2010-2013 Tinyboard Development Group */ -if (realpath($_SERVER['SCRIPT_FILENAME']) == str_replace('\\', '/', __FILE__)) { - // You cannot request this file directly. - exit; -} +defined('TINYBOARD') or exit; class PreparedQueryDebug { protected $query; diff --git a/inc/events.php b/inc/events.php index cc330dfb..e5a1e036 100644 --- a/inc/events.php +++ b/inc/events.php @@ -4,10 +4,7 @@ * Copyright (c) 2010-2013 Tinyboard Development Group */ -if (realpath($_SERVER['SCRIPT_FILENAME']) == str_replace('\\', '/', __FILE__)) { - // You cannot request this file directly. - exit; -} +defined('TINYBOARD') or exit; function event() { global $events; diff --git a/inc/filters.php b/inc/filters.php index cd512821..253a86b6 100644 --- a/inc/filters.php +++ b/inc/filters.php @@ -4,10 +4,7 @@ * Copyright (c) 2010-2013 Tinyboard Development Group */ -if (realpath($_SERVER['SCRIPT_FILENAME']) == str_replace('\\', '/', __FILE__)) { - // You cannot request this file directly. - exit; -} +defined('TINYBOARD') or exit; class Filter { private $condition; diff --git a/inc/functions.php b/inc/functions.php index 918a6ea9..9601c0f8 100644 --- a/inc/functions.php +++ b/inc/functions.php @@ -9,6 +9,8 @@ if (realpath($_SERVER['SCRIPT_FILENAME']) == str_replace('\\', '/', __FILE__)) { exit; } +define('TINYBOARD', null); + $microtime_start = microtime(true); require_once 'inc/display.php'; diff --git a/inc/image.php b/inc/image.php index d3692a39..1d591c25 100644 --- a/inc/image.php +++ b/inc/image.php @@ -4,10 +4,7 @@ * Copyright (c) 2010-2013 Tinyboard Development Group */ -if (realpath($_SERVER['SCRIPT_FILENAME']) == str_replace('\\', '/', __FILE__)) { - // You cannot request this file directly. - exit; -} +defined('TINYBOARD') or exit; class Image { public $src, $format, $image, $size; diff --git a/inc/mod/auth.php b/inc/mod/auth.php index 0733646f..f2003dfe 100644 --- a/inc/mod/auth.php +++ b/inc/mod/auth.php @@ -4,10 +4,7 @@ * Copyright (c) 2010-2013 Tinyboard Development Group */ -if (realpath($_SERVER['SCRIPT_FILENAME']) == str_replace('\\', '/', __FILE__)) { - // You cannot request this file directly. - exit; -} +defined('TINYBOARD') or exit; // create a hash/salt pair for validate logins function mkhash($username, $password, $salt = false) { diff --git a/inc/mod/pages.php b/inc/mod/pages.php index 99bfcf9d..5e45cdae 100644 --- a/inc/mod/pages.php +++ b/inc/mod/pages.php @@ -4,10 +4,7 @@ * Copyright (c) 2010-2013 Tinyboard Development Group */ -if (realpath($_SERVER['SCRIPT_FILENAME']) == str_replace('\\', '/', __FILE__)) { - // You cannot request this file directly. - exit; -} +defined('TINYBOARD') or exit; function mod_page($title, $template, $args, $subtitle = false) { global $config, $mod; diff --git a/inc/remote.php b/inc/remote.php index fbce8dab..6a685e82 100644 --- a/inc/remote.php +++ b/inc/remote.php @@ -4,10 +4,7 @@ * Copyright (c) 2010-2013 Tinyboard Development Group */ -if (realpath($_SERVER['SCRIPT_FILENAME']) == str_replace('\\', '/', __FILE__)) { - // You cannot request this file directly. - exit; -} +defined('TINYBOARD') or exit; class Remote { public function __construct($config) { diff --git a/inc/template.php b/inc/template.php index 378f36d0..c10ac454 100644 --- a/inc/template.php +++ b/inc/template.php @@ -4,10 +4,7 @@ * Copyright (c) 2010-2013 Tinyboard Development Group */ -if (realpath($_SERVER['SCRIPT_FILENAME']) == str_replace('\\', '/', __FILE__)) { - // You cannot request this file directly. - exit; -} +defined('TINYBOARD') or exit; $twig = false; From 14ff0fbeb3fd93e94a804d647971b8a9551c327f Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Fri, 6 Sep 2013 20:12:34 +1000 Subject: [PATCH 05/77] inc/mod.php has been irrelevant for a while. Time to remove it. --- inc/mod.php | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 inc/mod.php diff --git a/inc/mod.php b/inc/mod.php deleted file mode 100644 index 7b82c5d4..00000000 --- a/inc/mod.php +++ /dev/null @@ -1,15 +0,0 @@ - Date: Fri, 6 Sep 2013 23:09:18 +1000 Subject: [PATCH 06/77] Better and faster basic flood prevention, while merging it into $config['filters']. --- inc/config.php | 177 +++++++++++++++++++++++++++++----------------- inc/filters.php | 109 ++++++++++++++++++++++++++-- inc/functions.php | 58 +++++++-------- install.php | 17 ++++- install.sql | 21 ++++++ post.php | 23 +++--- 6 files changed, 294 insertions(+), 111 deletions(-) diff --git a/inc/config.php b/inc/config.php index e8b65967..81cae233 100644 --- a/inc/config.php +++ b/inc/config.php @@ -168,13 +168,6 @@ * ==================== */ - // Minimum time between between each post by the same IP address. - $config['flood_time'] = 10; - // Minimum time between between each post with the exact same content AND same IP address. - $config['flood_time_ip'] = 120; - // Same as above but by a different IP address. (Same content, not necessarily same IP address.) - $config['flood_time_same'] = 30; - /* * To further prevent spam and abuse, you can use DNS blacklists (DNSBL). A DNSBL is a list of IP * addresses published through the Internet Domain Name Service (DNS) either as a zone file that can be @@ -283,6 +276,118 @@ $config['recaptcha_public'] = '6LcXTcUSAAAAAKBxyFWIt2SO8jwx4W7wcSMRoN3f'; $config['recaptcha_private'] = '6LcXTcUSAAAAAOGVbVdhmEM1_SyRF4xTKe8jbzf_'; + /* + * Custom filters detect certain posts and reject/ban accordingly. They are made up of a condition and an + * action (for when ALL conditions are met). As every single post has to be put through each filter, + * having hundreds probably isn't ideal as it could slow things down. + * + * By default, the custom filters array is populated with basic flood prevention conditions. This + * includes forcing users to wait at least 5 seconds between posts. To disable (or amend) these flood + * prevention settings, you will need to empty the $config['filters'] array first. You can do so by + * adding "$config['filters'] = array();" to inc/instance-config.php. Basic flood prevention used to be + * controlled solely by config variables such as $config['flood_time'] and $config['flood_time_ip'], and + * it still is, as long as you leave the relevant $config['filters'] intact. These old config variables + * still exist for backwards-compatability and general convenience. + * + * Read more: http://tinyboard.org/docs/index.php?p=Config/Filters + */ + + // Minimum time between between each post by the same IP address. + $config['flood_time'] = 10; + // Minimum time between between each post with the exact same content AND same IP address. + $config['flood_time_ip'] = 120; + // Same as above but by a different IP address. (Same content, not necessarily same IP address.) + $config['flood_time_same'] = 30; + + // Minimum time between posts by the same IP address (all boards). + $config['filters'][] = array( + 'condition' => array( + 'flood-match' => array('ip'), // Only match IP address + 'flood-time' => &$config['flood_time'] // 10 seconds minimum + ), + 'action' => 'reject', + 'message' => &$config['error']['flood'] + ); + + // Minimum time between posts by the same IP address with the same text. + $config['filters'][] = array( + 'condition' => array( + 'flood-match' => array('ip', 'body'), // Match IP address and post body + 'flood-time' => &$config['flood_time_ip'] // 2 minutes minimum + ), + 'action' => 'reject', + 'message' => &$config['error']['flood'] + ); + + // Minimum time between posts with the same text. (Same content, but not always the same IP address.) + $config['filters'][] = array( + 'condition' => array( + 'flood-match' => array('body'), // Match IP address and post body + 'flood-time' => &$config['flood_time_same'] // 30 seconds minimum + ), + 'action' => 'reject', + 'message' => &$config['error']['flood'] + ); + + // Example: Minimum time between posts with the same file hash. + // $config['filters'][] = array( + // 'condition' => array( + // 'flood-match' => array('file'), // Match file hash + // 'flood-time' => 60 * 2 // 2 minutes minimum + // ), + // 'action' => 'reject', + // 'message' => &$config['error']['flood'] + // ); + + // An example of blocking an imaginary known spammer, who keeps posting a reply with the name "surgeon", + // ending his posts with "regards, the surgeon" or similar. + // $config['filters'][] = array( + // 'condition' => array( + // 'name' => '/^surgeon$/', + // 'body' => '/regards,\s+(the )?surgeon$/i', + // 'OP' => false + // ), + // 'action' => 'reject', + // 'message' => 'Go away, spammer.' + // ); + + // Same as above, but issuing a 3-hour ban instead of just reject the post. + // $config['filters'][] = array( + // 'condition' => array( + // 'name' => '/^surgeon$/', + // 'body' => '/regards,\s+(the )?surgeon$/i', + // 'OP' => false + // ), + // 'action' => 'ban', + // 'expires' => 60 * 60 * 3, // 3 hours + // 'reason' => 'Go away, spammer.' + // ); + + // PHP 5.3+ (anonymous functions) + // There is also a "custom" condition, making the possibilities of this feature pretty much endless. + // This is a bad example, because there is already a "name" condition built-in. + // $config['filters'][] = array( + // 'condition' => array( + // 'body' => '/h$/i', + // 'OP' => false, + // 'custom' => function($post) { + // if($post['name'] == 'Anonymous') + // return true; + // else + // return false; + // } + // ), + // 'action' => 'reject' + // ); + + // Filter flood prevention conditions ("flood-match") depend on a table which contains a cache of recent + // posts across all boards. This table is automatically purged of older posts, determining the maximum + // "age" by looking at each filter. However, when determining the maximum age, Tinyboard does not look + // outside the current board. This means that if you have a special flood condition for a specific board + // (contained in a board configuration file) which has a flood-time greater than any of those in the + // global configuration, you need to set the following variable to the maximum flood-time condition value. + // $config['flood_cache'] = 60 * 60 * 24; // 24 hours + /* * ==================== * Post settings @@ -401,57 +506,6 @@ // Require users to see the ban page at least once for a ban even if it has since expired. $config['require_ban_view'] = true; - /* - * Custom filters detect certain posts and reject/ban accordingly. They are made up of a - * condition and an action (for when ALL conditions are met). As every single post has to - * be put through each filter, having hundreds probably isn’t ideal as it could slow things down. - * - * Read more: http://tinyboard.org/docs/index.php?p=Config/Filters - * - * This used to be named $config['flood_filters'] (still exists as an alias). - */ - - // An example of blocking an imaginary known spammer, who keeps posting a reply with the name "surgeon", - // ending his posts with "regards, the surgeon" or similar. - // $config['filters'][] = array( - // 'condition' => array( - // 'name' => '/^surgeon$/', - // 'body' => '/regards,\s+(the )?surgeon$/i', - // 'OP' => false - // ), - // 'action' => 'reject', - // 'message' => 'Go away, spammer.' - // ); - - // Same as above, but issuing a 3-hour ban instead of just reject the post. - // $config['filters'][] = array( - // 'condition' => array( - // 'name' => '/^surgeon$/', - // 'body' => '/regards,\s+(the )?surgeon$/i', - // 'OP' => false - // ), - // 'action' => 'ban', - // 'expires' => 60 * 60 * 3, // 3 hours - // 'reason' => 'Go away, spammer.' - // ); - - // PHP 5.3+ (anonymous functions) - // There is also a "custom" condition, making the possibilities of this feature pretty much endless. - // This is a bad example, because there is already a "name" condition built-in. - // $config['filters'][] = array( - // 'condition' => array( - // 'body' => '/h$/i', - // 'OP' => false, - // 'custom' => function($post) { - // if($post['name'] == 'Anonymous') - // return true; - // else - // return false; - // } - // ), - // 'action' => 'reject' - // ); - /* * ==================== * Markup settings @@ -593,10 +647,6 @@ // that as a thumbnail instead of resizing/redrawing. $config['minimum_copy_resize'] = false; - // Image hashing function. There's really no reason to change this. - // sha1_file, md5_file, etc. You can also define your own similar function. - $config['file_hash'] = 'sha1_file'; - // Maximum image upload size in bytes. $config['max_filesize'] = 10 * 1024 * 1024; // 10MB // Maximum image dimensions. @@ -1156,7 +1206,8 @@ // Post bypass unoriginal content check on robot-enabled boards $config['mod']['postunoriginal'] = ADMIN; // Bypass flood check - $config['mod']['flood'] = ADMIN; + $config['mod']['bypass_filters'] = ADMIN; + $config['mod']['flood'] = &$config['mod']['bypass_filters']; // Raw HTML posting $config['mod']['rawhtml'] = ADMIN; diff --git a/inc/filters.php b/inc/filters.php index 253a86b6..81a33116 100644 --- a/inc/filters.php +++ b/inc/filters.php @@ -7,6 +7,7 @@ defined('TINYBOARD') or exit; class Filter { + public $flood_check; private $condition; public function __construct(array $arr) { @@ -22,6 +23,56 @@ class Filter { if (!is_callable($match)) error('Custom condition for filter is not callable!'); return $match($post); + case 'flood-match': + if (!is_array($match)) + error('Filter condition "flood-match" must be an array.'); + + // Filter out "flood" table entries which do not match this filter. + + $flood_check_matched = array(); + + foreach ($this->flood_check as $flood_post) { + foreach ($match as $flood_match_arg) { + switch ($flood_match_arg) { + case 'ip': + if ($flood_post['ip'] != $_SERVER['REMOTE_ADDR']) + continue 3; + break; + case 'body': + if ($flood_post['posthash'] != md5($post['body_nomarkup'])) + continue 3; + break; + case 'file': + if (!isset($post['filehash'])) + return false; + if ($flood_post['filehash'] != $post['filehash']) + continue 3; + break; + case 'board': + if ($flood_post['board'] != $post['board']) + continue 3; + break; + case 'isreply': + if ($flood_post['isreply'] == $post['op']) + continue 3; + break; + default: + error('Invalid filter flood condition: ' . $flood_match_arg); + } + } + $flood_check_matched[] = $flood_post; + } + + $this->flood_check = $flood_check_matched; + + return !empty($this->flood_check); + case 'flood-time': + foreach ($this->flood_check as $flood_post) { + if (time() - $flood_post['time'] <= $match) { + return true; + } + } + return false; case 'name': return preg_match($match, $post['name']); case 'trip': @@ -31,7 +82,7 @@ class Filter { case 'subject': return preg_match($match, $post['subject']); case 'body': - return preg_match($match, $post['body']); + return preg_match($match, $post['body_nomarkup']); case 'filename': if (!$post['has_file']) return false; @@ -126,14 +177,64 @@ class Filter { } } +function purge_flood_table() { + global $config; + + // Determine how long we need to keep a cache of posts for flood prevention. Unfortunately, it is not + // aware of flood filters in other board configurations. You can solve this problem by settings the + // config variable $config['flood_cache'] (seconds). + + if (isset($config['flood_cache'])) { + $max_time = &$config['flood_cache']; + } else { + $max_time = 0; + foreach ($config['filters'] as $filter) { + if (isset($filter['condition']['flood-time'])) + $max_time = max($max_time, $filter['condition']['flood-time']); + } + } + + $time = time() - $max_time; + + query("DELETE FROM ``flood`` WHERE ``time`` < $time") or error(db_error()); +} + function do_filters(array $post) { global $config; - if (!isset($config['filters'])) + if (!isset($config['filters']) || empty($config['filters'])) return; - foreach ($config['filters'] as $arr) { - $filter = new Filter($arr); + foreach ($config['filters'] as $filter) { + if (isset($filter['condition']['flood-match'])) { + $has_flood = true; + break; + } + } + + purge_flood_table(); + + if (isset($has_flood)) { + if ($post['has_file']) { + $query = prepare("SELECT * FROM ``flood`` WHERE `ip` = :ip OR `posthash` = :posthash OR `filehash` = :filehash"); + $query->bindValue(':ip', $_SERVER['REMOTE_ADDR']); + $query->bindValue(':posthash', md5($post['body_nomarkup'])); + $query->bindValue(':filehash', $post['filehash']); + } else { + $query = prepare("SELECT * FROM ``flood`` WHERE `ip` = :ip OR `posthash` = :posthash"); + $query->bindValue(':ip', $_SERVER['REMOTE_ADDR']); + $query->bindValue(':posthash', md5($post['body_nomarkup'])); + + } + $query->execute() or error(db_error($query)); + $flood_check = $query->fetchAll(PDO::FETCH_ASSOC); + } else { + $flood_check = false; + } + + foreach ($config['filters'] as $filter_array) { + $filter = new Filter($filter_array); + $filter->flood_check = $flood_check; if ($filter->check($post)) $filter->action(); } diff --git a/inc/functions.php b/inc/functions.php index 9601c0f8..a6963c91 100644 --- a/inc/functions.php +++ b/inc/functions.php @@ -572,30 +572,6 @@ function listBoards() { return $boards; } -function checkFlood($post) { - global $board, $config; - - $query = prepare(sprintf("SELECT COUNT(*) FROM ``posts_%s`` WHERE - (`ip` = :ip AND `time` >= :floodtime) - OR - (`ip` = :ip AND :body != '' AND `body_nomarkup` = :body AND `time` >= :floodsameiptime) - OR - (:body != '' AND `body_nomarkup` = :body AND `time` >= :floodsametime) LIMIT 1", $board['uri'])); - $query->bindValue(':ip', $_SERVER['REMOTE_ADDR']); - $query->bindValue(':body', $post['body']); - $query->bindValue(':floodtime', time()-$config['flood_time'], PDO::PARAM_INT); - $query->bindValue(':floodsameiptime', time()-$config['flood_time_ip'], PDO::PARAM_INT); - $query->bindValue(':floodsametime', time()-$config['flood_time_same'], PDO::PARAM_INT); - $query->execute() or error(db_error($query)); - - $flood = (bool) $query->fetchColumn(); - - if (event('check-flood', $post)) - return true; - - return $flood; -} - function until($timestamp) { $difference = $timestamp - time(); if ($difference < 60) { @@ -780,6 +756,22 @@ function threadExists($id) { return false; } +function insertFloodPost(array $post) { + global $board; + + $query = prepare("INSERT INTO ``flood`` VALUES (NULL, :ip, :board, :time, :posthash, :filehash, :isreply)"); + $query->bindValue(':ip', $_SERVER['REMOTE_ADDR']); + $query->bindValue(':board', $board['uri']); + $query->bindValue(':time', time()); + $query->bindValue(':posthash', md5($post['body_nomarkup'])); + if ($post['has_file']) + $query->bindValue(':filehash', $post['filehash']); + else + $query->bindValue(':filehash', null, PDO::PARAM_NULL); + $query->bindValue(':isreply', !$post['op']); + $query->execute() or error(db_error($query)); +} + function post(array $post) { global $pdo, $board; $query = prepare(sprintf("INSERT INTO ``posts_%s`` VALUES ( NULL, :thread, :subject, :email, :name, :trip, :capcode, :body, :body_nomarkup, :time, :time, :thumb, :thumbwidth, :thumbheight, :file, :width, :height, :filesize, :filename, :filehash, :password, :ip, :sticky, :locked, 0, :embed)", $board['uri'])); @@ -788,19 +780,19 @@ function post(array $post) { if (!empty($post['subject'])) { $query->bindValue(':subject', $post['subject']); } else { - $query->bindValue(':subject', NULL, PDO::PARAM_NULL); + $query->bindValue(':subject', null, PDO::PARAM_NULL); } if (!empty($post['email'])) { $query->bindValue(':email', $post['email']); } else { - $query->bindValue(':email', NULL, PDO::PARAM_NULL); + $query->bindValue(':email', null, PDO::PARAM_NULL); } if (!empty($post['trip'])) { $query->bindValue(':trip', $post['trip']); } else { - $query->bindValue(':trip', NULL, PDO::PARAM_NULL); + $query->bindValue(':trip', null, PDO::PARAM_NULL); } $query->bindValue(':name', $post['name']); @@ -811,27 +803,27 @@ function post(array $post) { $query->bindValue(':ip', isset($post['ip']) ? $post['ip'] : $_SERVER['REMOTE_ADDR']); if ($post['op'] && $post['mod'] && isset($post['sticky']) && $post['sticky']) { - $query->bindValue(':sticky', 1, PDO::PARAM_INT); + $query->bindValue(':sticky', true, PDO::PARAM_INT); } else { - $query->bindValue(':sticky', 0, PDO::PARAM_INT); + $query->bindValue(':sticky', false, PDO::PARAM_INT); } if ($post['op'] && $post['mod'] && isset($post['locked']) && $post['locked']) { - $query->bindValue(':locked', 1, PDO::PARAM_INT); + $query->bindValue(':locked', true, PDO::PARAM_INT); } else { - $query->bindValue(':locked', 0, PDO::PARAM_INT); + $query->bindValue(':locked', false, PDO::PARAM_INT); } if ($post['mod'] && isset($post['capcode']) && $post['capcode']) { $query->bindValue(':capcode', $post['capcode'], PDO::PARAM_INT); } else { - $query->bindValue(':capcode', NULL, PDO::PARAM_NULL); + $query->bindValue(':capcode', null, PDO::PARAM_NULL); } if (!empty($post['embed'])) { $query->bindValue(':embed', $post['embed']); } else { - $query->bindValue(':embed', NULL, PDO::PARAM_NULL); + $query->bindValue(':embed', null, PDO::PARAM_NULL); } if ($post['op']) { diff --git a/install.php b/install.php index e8bdda8b..0011a274 100644 --- a/install.php +++ b/install.php @@ -1,7 +1,7 @@ Date: Fri, 6 Sep 2013 23:10:25 +1000 Subject: [PATCH 07/77] CHARACTER SET not needed here --- install.php | 2 +- install.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/install.php b/install.php index 0011a274..45244b90 100644 --- a/install.php +++ b/install.php @@ -395,7 +395,7 @@ if (file_exists($config['has_installed'])) { case 'v0.9.6-dev-18': query("CREATE TABLE IF NOT EXISTS ``flood`` ( `id` int(11) unsigned NOT NULL AUTO_INCREMENT, - `ip` varchar(39) CHARACTER SET ascii NOT NULL, + `ip` varchar(39) NOT NULL, `board` varchar(58) CHARACTER SET utf8 NOT NULL, `time` int(11) NOT NULL, `posthash` char(32) NOT NULL, diff --git a/install.sql b/install.sql index 3c8cc886..25c0a0f1 100644 --- a/install.sql +++ b/install.sql @@ -253,7 +253,7 @@ CREATE TABLE IF NOT EXISTS `theme_settings` ( CREATE TABLE IF NOT EXISTS `flood` ( `id` int(11) unsigned NOT NULL AUTO_INCREMENT, - `ip` varchar(39) CHARACTER SET ascii NOT NULL, + `ip` varchar(39) NOT NULL, `board` varchar(58) CHARACTER SET utf8 NOT NULL, `time` int(11) NOT NULL, `posthash` char(32) NOT NULL, From ecda7abe92cf38ecb3ccf55ea6ea7a6bbfadac33 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Sat, 7 Sep 2013 00:04:22 +1000 Subject: [PATCH 08/77] bugfix lol --- inc/functions.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inc/functions.php b/inc/functions.php index a6963c91..df3a4eeb 100644 --- a/inc/functions.php +++ b/inc/functions.php @@ -768,7 +768,7 @@ function insertFloodPost(array $post) { $query->bindValue(':filehash', $post['filehash']); else $query->bindValue(':filehash', null, PDO::PARAM_NULL); - $query->bindValue(':isreply', !$post['op']); + $query->bindValue(':isreply', !$post['op'], PDO::PARAM_INT); $query->execute() or error(db_error($query)); } From 7b1a08d85cf28d46da459bb78b05a5bb8696fcce Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Sat, 7 Sep 2013 02:57:42 +1000 Subject: [PATCH 09/77] purge flood cache table after filter stuff, not before --- inc/filters.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/inc/filters.php b/inc/filters.php index 81a33116..212118bf 100644 --- a/inc/filters.php +++ b/inc/filters.php @@ -212,8 +212,6 @@ function do_filters(array $post) { } } - purge_flood_table(); - if (isset($has_flood)) { if ($post['has_file']) { $query = prepare("SELECT * FROM ``flood`` WHERE `ip` = :ip OR `posthash` = :posthash OR `filehash` = :filehash"); @@ -224,7 +222,6 @@ function do_filters(array $post) { $query = prepare("SELECT * FROM ``flood`` WHERE `ip` = :ip OR `posthash` = :posthash"); $query->bindValue(':ip', $_SERVER['REMOTE_ADDR']); $query->bindValue(':posthash', md5($post['body_nomarkup'])); - } $query->execute() or error(db_error($query)); $flood_check = $query->fetchAll(PDO::FETCH_ASSOC); @@ -238,5 +235,7 @@ function do_filters(array $post) { if ($filter->check($post)) $filter->action(); } + + purge_flood_table(); } From 9ccf62bb613d00d5d86391d90b7b2410f79c3da8 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Sat, 7 Sep 2013 03:09:52 +1000 Subject: [PATCH 10/77] yeah --- inc/cache.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inc/cache.php b/inc/cache.php index 95b1281f..d1200919 100644 --- a/inc/cache.php +++ b/inc/cache.php @@ -132,7 +132,7 @@ class Cache { case 'apc': return apc_clear_cache('user'); case 'php': - self::$cache[$key] = array(); + self::$cache = array(); break; case 'redis': if (!self::$cache) From e9ccc5d72d121859c8a766bd64a87d45d42f62a2 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Sat, 7 Sep 2013 12:40:35 +1000 Subject: [PATCH 11/77] Optionally EXPLAIN all SQL queries when in debug mode --- inc/config.php | 2 ++ inc/database.php | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/inc/config.php b/inc/config.php index 81cae233..43550dc3 100644 --- a/inc/config.php +++ b/inc/config.php @@ -44,6 +44,8 @@ $config['debug'] = false; // For development purposes. Displays (and "dies" on) all errors and warnings. Turn on with the above. $config['verbose_errors'] = true; + // EXPLAIN all SQL queries (when in debug mode). + $config['debug_explain'] = false; // Directory where temporary files will be created. $config['tmp'] = sys_get_temp_dir(); diff --git a/inc/database.php b/inc/database.php index d2f1af04..2c60015a 100644 --- a/inc/database.php +++ b/inc/database.php @@ -7,21 +7,29 @@ defined('TINYBOARD') or exit; class PreparedQueryDebug { - protected $query; + protected $query, $explain_query = false; public function __construct($query) { - global $pdo; + global $pdo, $config; $query = preg_replace("/[\n\t]+/", ' ', $query); $this->query = $pdo->prepare($query); + if ($config['debug'] && $config['debug_explain'] && preg_match('/^(SELECT|INSERT|UPDATE|DELETE) /', $query)) + $this->explain_query = $pdo->prepare("EXPLAIN $query"); } public function __call($function, $args) { global $config, $debug; if ($config['debug'] && $function == 'execute') { + if ($this->explain_query) { + $this->explain_query->execute() or error(db_error($explain_query)); + } $start = microtime(true); } + if ($this->explain_query && $function == 'bindValue') + call_user_func_array(array($this->explain_query, $function), $args); + $return = call_user_func_array(array($this->query, $function), $args); if ($config['debug'] && $function == 'execute') { @@ -29,6 +37,7 @@ class PreparedQueryDebug { $debug['sql'][] = array( 'query' => $this->query->queryString, 'rows' => $this->query->rowCount(), + 'explain' => $this->explain_query ? $this->explain_query->fetchAll(PDO::FETCH_ASSOC) : null, 'time' => '~' . round($time * 1000, 2) . 'ms' ); $debug['time']['db_queries'] += $time; @@ -118,6 +127,9 @@ function query($query) { sql_open(); if ($config['debug']) { + if ($config['debug_explain'] && preg_match('/^(SELECT|INSERT|UPDATE|DELETE) /', $query)) { + $explain = $pdo->query("EXPLAIN $query") or error(db_error()); + } $start = microtime(true); $query = $pdo->query($query); if (!$query) @@ -126,6 +138,7 @@ function query($query) { $debug['sql'][] = array( 'query' => $query->queryString, 'rows' => $query->rowCount(), + 'explain' => isset($explain) ? $explain->fetchAll(PDO::FETCH_ASSOC) : null, 'time' => '~' . round($time * 1000, 2) . 'ms' ); $debug['time']['db_queries'] += $time; From 55dc5cedc338f56268d8e302cdc52e97206543de Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Sat, 7 Sep 2013 12:50:32 +1000 Subject: [PATCH 12/77] Steal make_comment_hex() from plainib --- inc/filters.php | 6 +++--- inc/functions.php | 22 +++++++++++++++++++++- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/inc/filters.php b/inc/filters.php index 212118bf..27b167ea 100644 --- a/inc/filters.php +++ b/inc/filters.php @@ -39,7 +39,7 @@ class Filter { continue 3; break; case 'body': - if ($flood_post['posthash'] != md5($post['body_nomarkup'])) + if ($flood_post['posthash'] != make_comment_hex($post['body_nomarkup'])) continue 3; break; case 'file': @@ -216,12 +216,12 @@ function do_filters(array $post) { if ($post['has_file']) { $query = prepare("SELECT * FROM ``flood`` WHERE `ip` = :ip OR `posthash` = :posthash OR `filehash` = :filehash"); $query->bindValue(':ip', $_SERVER['REMOTE_ADDR']); - $query->bindValue(':posthash', md5($post['body_nomarkup'])); + $query->bindValue(':posthash', make_comment_hex($post['body_nomarkup'])); $query->bindValue(':filehash', $post['filehash']); } else { $query = prepare("SELECT * FROM ``flood`` WHERE `ip` = :ip OR `posthash` = :posthash"); $query->bindValue(':ip', $_SERVER['REMOTE_ADDR']); - $query->bindValue(':posthash', md5($post['body_nomarkup'])); + $query->bindValue(':posthash', make_comment_hex($post['body_nomarkup'])); } $query->execute() or error(db_error($query)); $flood_check = $query->fetchAll(PDO::FETCH_ASSOC); diff --git a/inc/functions.php b/inc/functions.php index df3a4eeb..ba1b04c8 100644 --- a/inc/functions.php +++ b/inc/functions.php @@ -763,7 +763,7 @@ function insertFloodPost(array $post) { $query->bindValue(':ip', $_SERVER['REMOTE_ADDR']); $query->bindValue(':board', $board['uri']); $query->bindValue(':time', time()); - $query->bindValue(':posthash', md5($post['body_nomarkup'])); + $query->bindValue(':posthash', make_comment_hex($post['body_nomarkup'])); if ($post['has_file']) $query->bindValue(':filehash', $post['filehash']); else @@ -1191,6 +1191,26 @@ function getPages($mod=false) { return $pages; } +// Stolen with permission from PlainIB (by Frank Usrs) +function make_comment_hex($str) { + // remove cross-board citations + // the numbers don't matter + $str = preg_replace('!>>>/[A-Za-z0-9]+/!', '', $str); + + if (function_exists('iconv')) { + // remove diacritics and other noise + // FIXME: this removes cyrillic entirely + $str = iconv('UTF-8', 'ASCII//TRANSLIT//IGNORE', $str); + } + + $str = strtolower($str); + + // strip all non-alphabet characters + $str = preg_replace('/[^a-z]/', '', $str); + + return md5($str); +} + function makerobot($body) { global $config; $body = strtolower($body); From b7f16dee0f4554f5e607e50f773af6852e716513 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Sat, 7 Sep 2013 12:58:23 +1000 Subject: [PATCH 13/77] Add ! syntax (NOT) to filters. Don't throttle duplicate post bodies when they are empty --- inc/config.php | 7 ++++--- inc/filters.php | 9 ++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/inc/config.php b/inc/config.php index 43550dc3..9f231e30 100644 --- a/inc/config.php +++ b/inc/config.php @@ -305,7 +305,7 @@ $config['filters'][] = array( 'condition' => array( 'flood-match' => array('ip'), // Only match IP address - 'flood-time' => &$config['flood_time'] // 10 seconds minimum + 'flood-time' => &$config['flood_time'] ), 'action' => 'reject', 'message' => &$config['error']['flood'] @@ -315,7 +315,8 @@ $config['filters'][] = array( 'condition' => array( 'flood-match' => array('ip', 'body'), // Match IP address and post body - 'flood-time' => &$config['flood_time_ip'] // 2 minutes minimum + 'flood-time' => &$config['flood_time_ip'], + '!body' => '/^$/', // Post body is NOT empty ), 'action' => 'reject', 'message' => &$config['error']['flood'] @@ -325,7 +326,7 @@ $config['filters'][] = array( 'condition' => array( 'flood-match' => array('body'), // Match IP address and post body - 'flood-time' => &$config['flood_time_same'] // 30 seconds minimum + 'flood-time' => &$config['flood_time_same'] ), 'action' => 'reject', 'message' => &$config['error']['flood'] diff --git a/inc/filters.php b/inc/filters.php index 27b167ea..5c934532 100644 --- a/inc/filters.php +++ b/inc/filters.php @@ -168,11 +168,14 @@ class Filter { public function check(array $post) { foreach ($this->condition as $condition => $value) { - if (!$this->match($post, $condition, $value)) + if ($condition[0] == '!') { + $NOT = true; + $condition = substr($condition, 1); + } else $NOT = false; + + if ($this->match($post, $condition, $value) == $NOT) return false; } - - /* match */ return true; } } From 0e23a6a2b43ccb388e7fcc62d5d73890c5bef019 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Sat, 7 Sep 2013 13:14:55 +1000 Subject: [PATCH 14/77] "flood filter" becomes "filter" --- inc/filters.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inc/filters.php b/inc/filters.php index 5c934532..d2802264 100644 --- a/inc/filters.php +++ b/inc/filters.php @@ -107,7 +107,7 @@ class Filter { switch($this->action) { case 'reject': - error(isset($this->message) ? $this->message : 'Posting throttled by flood filter.'); + error(isset($this->message) ? $this->message : 'Posting throttled by filter.'); case 'ban': if (!isset($this->reason)) error('The ban action requires a reason.'); From d80af7d077a1a5446c188e3338aa7035f78a43d2 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Sun, 8 Sep 2013 13:35:02 +1000 Subject: [PATCH 15/77] Bugfix: Sometimes caching here fucks up. Not really sure why yet. --- inc/functions.php | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/inc/functions.php b/inc/functions.php index ba1b04c8..819a935b 100644 --- a/inc/functions.php +++ b/inc/functions.php @@ -1067,10 +1067,16 @@ function index($page, $mod=false) { while ($th = $query->fetch(PDO::FETCH_ASSOC)) { $thread = new Thread($th, $mod ? '?/' : $config['root'], $mod); - if ($config['cache']['enabled'] && $cached = cache::get("thread_index_{$board['uri']}_{$th['id']}")) { - $replies = $cached['replies']; - $omitted = $cached['omitted']; - } else { + if ($config['cache']['enabled']) { + $cached = cache::get("thread_index_{$board['uri']}_{$th['id']}"); + if (isset($cached['replies'], $cached['omitted'])) { + $replies = $cached['replies']; + $omitted = $cached['omitted']; + } else { + $cached = false; + } + } + if (!$cached) { $posts = prepare(sprintf("SELECT * FROM ``posts_%s`` WHERE `thread` = :id ORDER BY `id` DESC LIMIT :limit", $board['uri'])); $posts->bindValue(':id', $th['id']); $posts->bindValue(':limit', ($th['sticky'] ? $config['threads_preview_sticky'] : $config['threads_preview']), PDO::PARAM_INT); From a13571cdade07802fcf8c72cb0924cf455032a12 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Sun, 8 Sep 2013 14:59:43 +1000 Subject: [PATCH 16/77] Comment mistake --- inc/config.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inc/config.php b/inc/config.php index 9f231e30..0cdbe709 100644 --- a/inc/config.php +++ b/inc/config.php @@ -325,7 +325,7 @@ // Minimum time between posts with the same text. (Same content, but not always the same IP address.) $config['filters'][] = array( 'condition' => array( - 'flood-match' => array('body'), // Match IP address and post body + 'flood-match' => array('body'), // Match only post body 'flood-time' => &$config['flood_time_same'] ), 'action' => 'reject', From d4cf4c7afb975cead41768c9e0d1960a46183409 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Sun, 8 Sep 2013 15:07:55 +1000 Subject: [PATCH 17/77] flood-count condition --- inc/config.php | 19 ++++++++++++++++--- inc/filters.php | 8 ++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/inc/config.php b/inc/config.php index 0cdbe709..0eac92b6 100644 --- a/inc/config.php +++ b/inc/config.php @@ -342,7 +342,20 @@ // 'message' => &$config['error']['flood'] // ); - // An example of blocking an imaginary known spammer, who keeps posting a reply with the name "surgeon", + // Example: Use the "flood-count" condition to only match if the user has made at least two posts with + // the same content and IP address in the past 2 minutes. + // $config['filters'][] = array( + // 'condition' => array( + // 'flood-match' => array('ip', 'body'), // Match IP address and post body + // 'flood-time' => 60 * 2, // 2 minutes + // 'flood-count' => 2 // At least two recent posts + // ), + // '!body' => '/^$/', + // 'action' => 'reject', + // 'message' => &$config['error']['flood'] + // ); + + // Example: Blocking an imaginary known spammer, who keeps posting a reply with the name "surgeon", // ending his posts with "regards, the surgeon" or similar. // $config['filters'][] = array( // 'condition' => array( @@ -354,7 +367,7 @@ // 'message' => 'Go away, spammer.' // ); - // Same as above, but issuing a 3-hour ban instead of just reject the post. + // Example: Same as above, but issuing a 3-hour ban instead of just reject the post. // $config['filters'][] = array( // 'condition' => array( // 'name' => '/^surgeon$/', @@ -366,7 +379,7 @@ // 'reason' => 'Go away, spammer.' // ); - // PHP 5.3+ (anonymous functions) + // Example: PHP 5.3+ (anonymous functions) // There is also a "custom" condition, making the possibilities of this feature pretty much endless. // This is a bad example, because there is already a "name" condition built-in. // $config['filters'][] = array( diff --git a/inc/filters.php b/inc/filters.php index d2802264..d5353afb 100644 --- a/inc/filters.php +++ b/inc/filters.php @@ -73,6 +73,14 @@ class Filter { } } return false; + case 'flood-count': + $count = 0; + foreach ($this->flood_check as $flood_post) { + if (time() - $flood_post['time'] <= $this->condition['flood-time']) { + ++$count; + } + } + return $count >= $match; case 'name': return preg_match($match, $post['name']); case 'trip': From 9a846d5ad5838d4db824623d9916d6b4e34c29ee Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Sun, 8 Sep 2013 17:01:55 +1000 Subject: [PATCH 18/77] Use Unicode in antispam stuff --- inc/anti-bot.php | 42 +++++++++++++++++++++++++++++------------- inc/config.php | 3 +++ 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/inc/anti-bot.php b/inc/anti-bot.php index dbe87e2b..59f5351f 100644 --- a/inc/anti-bot.php +++ b/inc/anti-bot.php @@ -11,14 +11,19 @@ $hidden_inputs_twig = array(); class AntiBot { public $salt, $inputs = array(), $index = 0; - public static function randomString($length, $uppercase = false, $special_chars = false) { + public static function randomString($length, $uppercase = false, $special_chars = false, $unicode_chars = false) { $chars = 'abcdefghijklmnopqrstuvwxyz0123456789'; if ($uppercase) $chars .= 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'; if ($special_chars) $chars .= ' ~!@#$%^&*()_+,./;\'[]\\{}|:<>?=-` '; + if ($unicode_chars) { + $len = strlen($chars) / 10; + for ($n = 0; $n < $len; $n++) + $chars .= mb_convert_encoding('&#' . mt_rand(0x2600, 0x26FF) . ';', 'UTF-8', 'HTML-ENTITIES'); + } - $chars = str_split($chars); + $chars = preg_split('//u', $chars, -1, PREG_SPLIT_NO_EMPTY); $ch = array(); @@ -41,10 +46,10 @@ class AntiBot { } public static function make_confusing($string) { - $chars = str_split($string); + $chars = preg_split('//u', $string, -1, PREG_SPLIT_NO_EMPTY); foreach ($chars as &$c) { - if (rand(0, 2) != 0) + if (mt_rand(0, 3) != 0) $c = utf8tohtml($c); else $c = mb_encode_numericentity($c, array(0, 0xffff, 0, 0xffff), 'UTF-8'); @@ -65,13 +70,13 @@ class AntiBot { shuffle($config['spam']['hidden_input_names']); - $input_count = rand($config['spam']['hidden_inputs_min'], $config['spam']['hidden_inputs_max']); + $input_count = mt_rand($config['spam']['hidden_inputs_min'], $config['spam']['hidden_inputs_max']); $hidden_input_names_x = 0; for ($x = 0; $x < $input_count ; $x++) { - if ($hidden_input_names_x === false || rand(0, 2) == 0) { + if ($hidden_input_names_x === false || mt_rand(0, 2) == 0) { // Use an obscure name - $name = $this->randomString(rand(10, 40)); + $name = $this->randomString(mt_rand(10, 40), false, false, $config['spam']['unicode']); } else { // Use a pre-defined confusing name $name = $config['spam']['hidden_input_names'][$hidden_input_names_x++]; @@ -79,25 +84,33 @@ class AntiBot { $hidden_input_names_x = false; } - if (rand(0, 2) == 0) { + if (mt_rand(0, 2) == 0) { // Value must be null $this->inputs[$name] = ''; - } elseif (rand(0, 4) == 0) { + } elseif (mt_rand(0, 4) == 0) { // Numeric value - $this->inputs[$name] = (string)rand(0, 100); + $this->inputs[$name] = (string)mt_rand(0, 100000); } else { // Obscure value - $this->inputs[$name] = $this->randomString(rand(5, 100), true, true); + $this->inputs[$name] = $this->randomString(mt_rand(5, 100), true, true, $config['spam']['unicode']); } } } + public static function space() { + if (mt_rand(0, 3) != 0) + return ' '; + return str_repeat(' ', mt_rand(1, 3)); + } + public function html($count = false) { global $config; $elements = array( '', '', + '', + '', '', '', '', @@ -110,7 +123,7 @@ class AntiBot { $html = ''; if ($count === false) { - $count = rand(1, count($this->inputs) / 15); + $count = mt_rand(1, abs(count($this->inputs) / 15) + 1); } if ($count === true) { @@ -125,6 +138,9 @@ class AntiBot { $element = false; while (!$element) { $element = $elements[array_rand($elements)]; + $element = str_replace(' ', self::space(), $element); + if (mt_rand(0, 5) == 0) + $element = str_replace('>', self::space() . '>', $element); if (strpos($element, 'textarea') !== false && $value == '') { // There have been some issues with mobile web browsers and empty