preliminary inbound nntpchan support

inc/config.php

@@ -1689,6 +1689,23 @@
 	// Example: Adding the pre-markup post body to the API as "com_nomarkup".
 	// $config['api']['extra_fields'] = array('body_nomarkup' => 'com_nomarkup');
 
+/*
+ * ==================
+ *  NNTPChan settings
+ * ==================
+ */
+
+	$config['nntpchan'] = array();
+
+	// Enable NNTPChan integration
+	$config['nntpchan']['enabled'] = false;
+
+	// NNTP server
+	$config['nntpchan']['server'] = "localhost:1119";
+
+	// Global dispatch array. Add your boards to it to enable them.
+	$config['nntpchan']['dispatch'] = array(); // 'overchan.test' => 'test'
+
 /*
  * ====================
  *  Other/uncategorized

install.php

@@ -1,7 +1,7 @@
 <?php
 
 // Installation/upgrade file	
-define('VERSION', '5.1.2');
+define('VERSION', '5.1.3');
 
 require 'inc/functions.php';
 
@@ -574,6 +574,19 @@ if (file_exists($config['has_installed'])) {
                         foreach ($boards as &$board) {
                                 query(sprintf("ALTER TABLE ``posts_%s`` ADD `cycle` int(1) NOT NULL AFTER `locked`", $board['uri'])) or error(db_error());
                         }
+		case '5.1.2':
+			query('CREATE TABLE ``nntp_references`` (
+				  `board` varchar(60) NOT NULL,
+				  `id` int(11) unsigned NOT NULL,
+				  `message_id` varchar(255) CHARACTER SET ascii NOT NULL,
+				  `message_id_digest` varchar(40) CHARACTER SET ascii NOT NULL,
+				  `own` tinyint(1) NOT NULL,
+				  `headers` text,
+				  PRIMARY KEY (`message_id_digest`),
+				  UNIQUE KEY `message_id` (`message_id`),
+				  UNIQUE KEY `u_board_id` (`board`, `id`)
+				) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+			') or error(db_error());
 
 		case false:
 			// TODO: enhance Tinyboard -> vichan upgrade path.

install.sql

@@ -314,6 +314,25 @@ CREATE TABLE `pages` (
   UNIQUE KEY `u_pages` (`name`,`board`)
 ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4;
 
+-- --------------------------------------------------------
+
+--
+-- Table structure for table `nntp_references`
+--
+
+CREATE TABLE `nntp_references` (
+  `board` varchar(60) NOT NULL,
+  `id` int(11) unsigned NOT NULL,
+  `message_id` varchar(255) CHARACTER SET ascii NOT NULL,
+  `message_id_digest` varchar(40) CHARACTER SET ascii NOT NULL,
+  `own` tinyint(1) NOT NULL,
+  `headers` text,
+  PRIMARY KEY (`message_id_digest`),
+  UNIQUE KEY `message_id` (`message_id`),
+  UNIQUE KEY `u_board_id` (`board`, `id`)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4;
+
+
 /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
 /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
 /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;

post.php

@@ -11,6 +11,160 @@ if ((!isset($_POST['mod']) || !$_POST['mod']) && $config['board_locked']) {
     error("Board is locked");
 }
 
+$dropped_post = false;
+
+if (isset($_GET['Newsgroups']) && $config['nntpchan']['enabled']) {
+	$_POST = array();
+	$_POST['json_response'] = true;
+
+	$headers = json_encode($_GET);
+
+	if (!isset ($_GET['Message-Id'])) {
+		if (!isset ($_GET['Message-ID'])) {
+			error("NNTPChan: No message ID");
+		}
+		else $msgid = $_GET['Message-ID'];
+	}
+	else $msgid = $_GET['Message-Id'];
+
+	$groups = preg_split("/,\s*/", $_GET['Newsgroups']);
+	if (count($groups) != 1) {
+		error("NNTPChan: Messages can go to only one newsgroup");
+	}
+	$group = $groups[0];
+
+	if (!isset($config['nntpchan']['dispatch'][$group])) {
+		error("NNTPChan: We don't synchronize $group");
+	}
+	$xboard = $config['nntpchan']['dispatch'][$group];
+
+	$ref = null;
+	if (isset ($_GET['References'])) {
+		$refs = preg_split("/,\s*/", $_GET['References']);
+
+		if (count($refs) > 1) {
+			error("NNTPChan: We don't support multiple references");
+		}
+
+		$ref = $refs[0];
+
+		$query = prepare("SELECT `board`,`id` FROM ``nntp_references`` WHERE `message_id` = :ref");
+                $query->bindValue(':ref', $ref);
+                $query->execute() or error(db_error($query));
+
+		$ary = $query->fetchAll(PDO::FETCH_ASSOC);
+
+		if (count($ary) == 0) {
+			error("NNTPChan: We don't have $ref that $msgid references");
+		}
+
+		$p_id = $ary[0]['id'];
+		$p_board = $ary[0]['board'];
+
+		if ($p_board != $xboard) {
+			error("NNTPChan: Cross board references not allowed. Tried to reference $p_board on $xboard");
+		}
+
+		$_POST['thread'] = $p_id;
+	}
+
+	$date = isset($_GET['Date']) ? strtotime($_GET['Date']) : time();
+
+	list($ct) = explode('; ', $_GET['Content-Type']);
+
+	$query = prepare("SELECT COUNT(*) AS `c` FROM ``nntp_references`` WHERE `message_id` = :msgid");
+	$query->bindValue(":msgid", $msgid);
+	$query->execute() or error(db_error($query));
+
+	$a = $query->fetch(PDO::FETCH_ASSOC);
+	if ($a['c'] > 0) {
+		error("NNTPChan: We already have this post. Post discarded.");
+	}
+
+	if ($ct == 'text/plain') {
+		$content = file_get_contents("php://input");
+	}
+	elseif ($ct == 'multipart/mixed' || $ct == 'multipart/form-data') {
+		_syslog(LOG_INFO, "MM: Files: ".print_r($GLOBALS, true));
+		$content = '';
+
+		$tmpfiles = $_FILES['attachment'];
+		foreach ($tmpfiles as $id => $file) {
+			if ($file['type'] == 'text/plain') {
+				$content .= file_get_contents($file['tmp_name']);
+				unset($_FILES['attachment'][$id]);
+			}
+			elseif ($file['type'] == 'message/rfc822') { // Signed message, ignore for now
+				unset($_FILES['attachment'][$id]);
+			}
+			else { // A real attachment :^)
+			}
+		}
+
+		$_FILES = $_FILES['attachment'];
+
+
+	}
+	else {
+		error("NNTPChan: Wrong mime type: $ct");
+	}
+
+	$_POST['subject'] = isset($_GET['Subject']) ? $_GET['Subject'] : '';
+	$_POST['board'] = $xboard;
+
+	if (isset ($_GET['From'])) {
+		list($name, $mail) = explode(" <", $_GET['From'], 2);
+		$mail = preg_replace('/>\s+$/', '', $mail);
+
+		$_POST['name'] = $name;
+		//$_POST['email'] = $mail;
+		$_POST['email'] = '';
+	}
+
+	if (isset ($_GET['X_Sage'])) {
+		$_POST['email'] = 'sage';
+	}
+
+	$content = preg_replace_callback('/>>([0-9a-fA-F]{6,})/', function($id) use ($xboard) {
+		$id = $id[1];
+
+		$query = prepare("SELECT `board`,`id` FROM ``nntp_references`` WHERE `message_id_digest` LIKE :rule");
+		$idx = $id . "%";
+                $query->bindValue(':rule', $idx);
+                $query->execute() or error(db_error($query));
+		
+		$ary = $query->fetchAll(PDO::FETCH_ASSOC);
+		if (count($ary) == 0) {
+			return ">>>>$id";
+		}
+		else {
+			$ret = array();
+			foreach ($ary as $v) {
+				if ($v['board'] != $xboard) {
+					$ret[] = ">>>/".$v['board']."/".$v['id'];
+				}
+				else {
+					$ret[] = ">>".$v['id'];
+				}
+			}
+			return implode($ret, ", ");
+		}
+	}, $content);
+
+	$_POST['body'] = $content;
+
+	$dropped_post = array(
+		'date' => $date,
+		'board' => $xboard,
+		'msgid' => $msgid,
+		'headers' => $headers,
+		'from_nntp' => true,
+	);
+}
+elseif (isset($_GET['Newsgroups'])) {
+	error("NNTPChan: NNTPChan support is disabled");
+}
+
 if (isset($_POST['delete'])) {
 	// Delete
 	
@@ -178,8 +332,8 @@ if (isset($_POST['delete'])) {
 		header('Content-Type: text/json');
 		echo json_encode(array('success' => true));
 	}
-} elseif (isset($_POST['post'])) {
+} elseif (isset($_POST['post']) || $dropped_post) {
-	if (!isset($_POST['body'], $_POST['board']))
+	if (!isset($_POST['body'], $_POST['board']) && !$dropped_post)
 		error($config['error']['bot']);
 
 	$post = array('board' => $_POST['board'], 'files' => array());
@@ -206,61 +360,67 @@ if (isset($_POST['delete'])) {
 	} else
 		$post['op'] = true;
 
-	// Check for CAPTCHA right after opening the board so the "return" link is in there
+
-	if ($config['recaptcha']) {
+	if (!$dropped_post) {
-		if (!isset($_POST['recaptcha_challenge_field']) || !isset($_POST['recaptcha_response_field']))
+		// Check for CAPTCHA right after opening the board so the "return" link is in there
+		if ($config['recaptcha']) {
+			if (!isset($_POST['recaptcha_challenge_field']) || !isset($_POST['recaptcha_response_field']))
+				error($config['error']['bot']);	
+				// Check what reCAPTCHA has to say...
+				$resp = recaptcha_check_answer($config['recaptcha_private'],
+				$_SERVER['REMOTE_ADDR'],
+				$_POST['recaptcha_challenge_field'],
+				$_POST['recaptcha_response_field']);
+			if (!$resp->is_valid) {
+				error($config['error']['captcha']);
+			}
+		}
+
+		if (!(($post['op'] && $_POST['post'] == $config['button_newtopic']) ||
+			(!$post['op'] && $_POST['post'] == $config['button_reply'])))
 			error($config['error']['bot']);
-		// Check what reCAPTCHA has to say...
-		$resp = recaptcha_check_answer($config['recaptcha_private'],
-			$_SERVER['REMOTE_ADDR'],
-			$_POST['recaptcha_challenge_field'],
-			$_POST['recaptcha_response_field']);
-		if (!$resp->is_valid) {
-			error($config['error']['captcha']);
-		}
-	}
-
-	if (!(($post['op'] && $_POST['post'] == $config['button_newtopic']) ||
-		(!$post['op'] && $_POST['post'] == $config['button_reply'])))
-		error($config['error']['bot']);
 	
-	// Check the referrer
+		// Check the referrer
-	if ($config['referer_match'] !== false &&
+		if ($config['referer_match'] !== false &&
-		(!isset($_SERVER['HTTP_REFERER']) || !preg_match($config['referer_match'], rawurldecode($_SERVER['HTTP_REFERER']))))
+			(!isset($_SERVER['HTTP_REFERER']) || !preg_match($config['referer_match'], rawurldecode($_SERVER['HTTP_REFERER']))))
-		error($config['error']['referer']);
+			error($config['error']['referer']);
 	
-	checkDNSBL();
+		checkDNSBL();
 		
-	// Check if banned
+		// Check if banned
-	checkBan($board['uri']);
+		checkBan($board['uri']);
 
-	if ($post['mod'] = isset($_POST['mod']) && $_POST['mod']) {
+		if ($post['mod'] = isset($_POST['mod']) && $_POST['mod']) {
-		check_login(false);
+			check_login(false);
-		if (!$mod) {
+			if (!$mod) {
-			// Liar. You're not a mod.
+				// Liar. You're not a mod.
-			error($config['error']['notamod']);
+				error($config['error']['notamod']);
+			}
+		
+			$post['sticky'] = $post['op'] && isset($_POST['sticky']);
+			$post['locked'] = $post['op'] && isset($_POST['lock']);
+			$post['raw'] = isset($_POST['raw']);
+		
+			if ($post['sticky'] && !hasPermission($config['mod']['sticky'], $board['uri']))
+				error($config['error']['noaccess']);
+			if ($post['locked'] && !hasPermission($config['mod']['lock'], $board['uri']))
+				error($config['error']['noaccess']);
+			if ($post['raw'] && !hasPermission($config['mod']['rawhtml'], $board['uri']))
+				error($config['error']['noaccess']);
 		}
 		
-		$post['sticky'] = $post['op'] && isset($_POST['sticky']);
+		if (!$post['mod']) {
-		$post['locked'] = $post['op'] && isset($_POST['lock']);
+			$post['antispam_hash'] = checkSpam(array($board['uri'], isset($post['thread']) ? $post['thread'] : ($config['try_smarter'] && isset($_POST['page']) ? 0 - (int)$_POST['page'] : null)));
-		$post['raw'] = isset($_POST['raw']);
+			if ($post['antispam_hash'] === true)
-		
+				error($config['error']['spam']);
-		if ($post['sticky'] && !hasPermission($config['mod']['sticky'], $board['uri']))
+		}
-			error($config['error']['noaccess']);
-		if ($post['locked'] && !hasPermission($config['mod']['lock'], $board['uri']))
-			error($config['error']['noaccess']);
-		if ($post['raw'] && !hasPermission($config['mod']['rawhtml'], $board['uri']))
-			error($config['error']['noaccess']);
-	}
 	
-	if (!$post['mod']) {
+		if ($config['robot_enable'] && $config['robot_mute']) {
-		$post['antispam_hash'] = checkSpam(array($board['uri'], isset($post['thread']) ? $post['thread'] : ($config['try_smarter'] && isset($_POST['page']) ? 0 - (int)$_POST['page'] : null)));
+			checkMute();
-		if ($post['antispam_hash'] === true)
+		}
-			error($config['error']['spam']);
 	}
-	
+	else {
-	if ($config['robot_enable'] && $config['robot_mute']) {
+		$mod = $post['mod'] = false;
-		checkMute();
 	}
 	
 	//Check if thread exists
@@ -371,28 +531,36 @@ if (isset($_POST['delete'])) {
 	$post['email'] = str_replace(' ', '%20', htmlspecialchars($_POST['email']));
 	$post['body'] = $_POST['body'];
 	$post['password'] = $_POST['password'];
-	$post['has_file'] = (!isset($post['embed']) && (($post['op'] && !isset($post['no_longer_require_an_image_for_op']) && $config['force_image_op']) || !empty($_FILES['file']['name'])));
+	$post['has_file'] = (!isset($post['embed']) && (($post['op'] && !isset($post['no_longer_require_an_image_for_op']) && $config['force_image_op']) || count($_FILES) > 0));
 	
-	if (!($post['has_file'] || isset($post['embed'])) || (($post['op'] && $config['force_body_op']) || (!$post['op'] && $config['force_body']))) {
+	if (!$dropped_post) {
-		$stripped_whitespace = preg_replace('/[\s]/u', '', $post['body']);
+
-		if ($stripped_whitespace == '') {
+		if (!($post['has_file'] || isset($post['embed'])) || (($post['op'] && $config['force_body_op']) || (!$post['op'] && $config['force_body']))) {
-			error($config['error']['tooshort_body']);
+			$stripped_whitespace = preg_replace('/[\s]/u', '', $post['body']);
+			if ($stripped_whitespace == '') {
+				error($config['error']['tooshort_body']);
+			}
+		}
+	
+		if (!$post['op']) {
+			// Check if thread is locked
+			// but allow mods to post
+			if ($thread['locked'] && !hasPermission($config['mod']['postinlocked'], $board['uri']))
+				error($config['error']['locked']);
+		
+			$numposts = numPosts($post['thread']);
+		
+			if ($config['reply_hard_limit'] != 0 && $config['reply_hard_limit'] <= $numposts['replies'])
+				error($config['error']['reply_hard_limit']);
+		
+			if ($post['has_file'] && $config['image_hard_limit'] != 0 && $config['image_hard_limit'] <= $numposts['images'])
+				error($config['error']['image_hard_limit']);
 		}
 	}
-	
+	else {
-	if (!$post['op']) {
+		if (!$post['op']) {
-		// Check if thread is locked
+                        $numposts = numPosts($post['thread']);
-		// but allow mods to post
+		}
-		if ($thread['locked'] && !hasPermission($config['mod']['postinlocked'], $board['uri']))
-			error($config['error']['locked']);
-		
-		$numposts = numPosts($post['thread']);
-		
-		if ($config['reply_hard_limit'] != 0 && $config['reply_hard_limit'] <= $numposts['replies'])
-			error($config['error']['reply_hard_limit']);
-		
-		if ($post['has_file'] && $config['image_hard_limit'] != 0 && $config['image_hard_limit'] <= $numposts['images'])
-			error($config['error']['image_hard_limit']);
 	}
 		
 	if ($post['has_file']) {
@@ -442,7 +610,7 @@ if (isset($_POST['delete'])) {
 	
 	$trip = generate_tripcode($post['name']);
 	$post['name'] = $trip[0];
-	$post['trip'] = isset($trip[1]) ? $trip[1] : '';
+	$post['trip'] = isset($trip[1]) ? $trip[1] : ''; // XX: Dropped posts and tripcodes
 	
 	$noko = false;
 	if (strtolower($post['email']) == 'noko') {
@@ -477,15 +645,17 @@ if (isset($_POST['delete'])) {
 
 	if (empty($post['files'])) $post['has_file'] = false;
 
-	// Check for a file
+	if (!$dropped_post) {
-	if ($post['op'] && !isset($post['no_longer_require_an_image_for_op'])) {
+		// Check for a file
-		if (!$post['has_file'] && $config['force_image_op'])
+		if ($post['op'] && !isset($post['no_longer_require_an_image_for_op'])) {
-			error($config['error']['noimage']);
+			if (!$post['has_file'] && $config['force_image_op'])
-	}
+				error($config['error']['noimage']);
+		}
 
-	// Check for too many files
+		// Check for too many files
-	if (sizeof($post['files']) > $config['max_images'])
+		if (sizeof($post['files']) > $config['max_images'])
-		error($config['error']['toomanyimages']);
+			error($config['error']['toomanyimages']);
+	}
 
 	if ($config['strip_combining_chars']) {
 		$post['name'] = strip_combining_chars($post['name']);
@@ -494,18 +664,19 @@ if (isset($_POST['delete'])) {
 		$post['body'] = strip_combining_chars($post['body']);
 	}
 	
-	// Check string lengths
+	if (!$dropped_post) {
-	if (mb_strlen($post['name']) > 35)
+		// Check string lengths
-		error(sprintf($config['error']['toolong'], 'name'));	
+		if (mb_strlen($post['name']) > 35)
-	if (mb_strlen($post['email']) > 40)
+			error(sprintf($config['error']['toolong'], 'name'));	
-		error(sprintf($config['error']['toolong'], 'email'));
+		if (mb_strlen($post['email']) > 40)
-	if (mb_strlen($post['subject']) > 100)
+			error(sprintf($config['error']['toolong'], 'email'));
-		error(sprintf($config['error']['toolong'], 'subject'));
+		if (mb_strlen($post['subject']) > 100)
-	if (!$mod && mb_strlen($post['body']) > $config['max_body'])
+			error(sprintf($config['error']['toolong'], 'subject'));
-		error($config['error']['toolong_body']);
+		if (!$mod && mb_strlen($post['body']) > $config['max_body'])
-	if (mb_strlen($post['password']) > 20)
+			error($config['error']['toolong_body']);
-		error(sprintf($config['error']['toolong'], 'password'));
+		if (mb_strlen($post['password']) > 20)
-		
+			error(sprintf($config['error']['toolong'], 'password'));
+	}
 	wordfilters($post['body']);
 	
 	$post['body'] = escape_markup_modifiers($post['body']);
@@ -514,6 +685,7 @@ if (isset($_POST['delete'])) {
 		$post['body'] .= "\n<tinyboard raw html>1</tinyboard>";
 	}
 	
+	if (!$dropped_post)
 	if (($config['country_flags'] && !$config['allow_no_country']) || ($config['country_flags'] && $config['allow_no_country'] && !isset($_POST['no_country']))) {
 		require 'inc/lib/geoip/geoip.inc';
 		$gi=geoip\geoip_open('inc/lib/geoip/GeoIPv6.dat', GEOIP_STANDARD);
@@ -555,6 +727,7 @@ if (isset($_POST['delete'])) {
 		$post['body'] .= "\n<tinyboard tag>" . $_POST['tag'] . "</tinyboard>";
 	}
 
+	if (!$dropped_post)
         if ($config['proxy_save'] && isset($_SERVER['HTTP_X_FORWARDED_FOR'])) {
 		$proxy = preg_replace("/[^0-9a-fA-F.,: ]/", '', $_SERVER['HTTP_X_FORWARDED_FOR']);
 		$post['body'] .= "\n<tinyboard proxy>".$proxy."</tinyboard>";
@@ -627,7 +800,7 @@ if (isset($_POST['delete'])) {
 		}
 	}
 
-	if (!hasPermission($config['mod']['bypass_filters'], $board['uri'])) {
+	if (!hasPermission($config['mod']['bypass_filters'], $board['uri']) && !$dropped_post) {
 		require_once 'inc/filters.php';
 
 		do_filters($post);
@@ -830,11 +1003,11 @@ if (isset($_POST['delete'])) {
 		}
 	
 	// Do filters again if OCRing
-	if ($config['tesseract_ocr'] && !hasPermission($config['mod']['bypass_filters'], $board['uri'])) {
+	if ($config['tesseract_ocr'] && !hasPermission($config['mod']['bypass_filters'], $board['uri']) && !$dropped_post) {
 		do_filters($post);
 	}
 
-	if (!hasPermission($config['mod']['postunoriginal'], $board['uri']) && $config['robot_enable'] && checkRobot($post['body_nomarkup'])) {
+	if (!hasPermission($config['mod']['postunoriginal'], $board['uri']) && $config['robot_enable'] && checkRobot($post['body_nomarkup']) && !$dropped_post) {
 		undoImage($post);
 		if ($config['robot_mute']) {
 			error(sprintf($config['error']['muted'], mute()));
@@ -873,6 +1046,18 @@ if (isset($_POST['delete'])) {
 	$post['id'] = $id = post($post);
 	$post['slug'] = slugify($post);
 	
+
+	if ($dropped_post && $dropped_post['from_nntp']) {
+	        $query = prepare("INSERT INTO ``nntp_references`` (`board`, `id`, `message_id`, `message_id_digest`, `own`, `headers`) VALUES ".
+	                                                         "(:board , :id , :message_id , :message_id_digest , false, :headers)");
+
+		$query->bindValue(':board', $dropped_post['board']);
+		$query->bindValue(':id', $id);
+		$query->bindValue(':message_id', $dropped_post['msgid']);
+		$query->bindValue(':message_id_digest', sha1($dropped_post['msgid']));
+		$query->bindValue(':headers', $dropped_post['headers']);
+		$query->execute() or error(db_error($query));
+	}
 	insertFloodPost($post);
 
 	// Handle cyclical threads