Preserve official links and extract legacy package tables before review
This commit is contained in:
hellcat
parent
efcd02bef3
commit
b7831241f8
@ -9,6 +9,7 @@ final class ArticleArtifact
|
||||
public function build(array $data): array
|
||||
{
|
||||
validator($data, ['source_key' => 'required|string|max:160', 'revision' => 'required|string|size:64', 'generation' => 'required|integer|min:1', 'title' => 'required|string|max:200', 'text' => 'present|string|max:100000', 'sent_at' => 'required|date', 'images' => 'present|array|max:30', 'images.*.sha256' => 'required|regex:/^[a-f0-9]{64}$/D', 'images.*.mime' => 'required|in:image/png,image/jpeg,image/webp,image/gif', 'images.*.bytes' => 'required|integer|min:1|max:20971520'])->validate();
|
||||
validator($data, ['links' => 'sometimes|array|max:100', 'links.*.offset' => 'required|integer|min:0', 'links.*.length' => 'required|integer|min:1', 'links.*.url' => 'required|string|max:4096'])->validate();
|
||||
$code = 'telefox-'.substr(hash('sha256', $data['source_key']), 0, 24);
|
||||
$body = "<div v-pre>\n".implode("\n", array_map(fn ($s) => '<p>'.htmlspecialchars($s, ENT_QUOTES | ENT_SUBSTITUTE, 'UTF-8').'</p>', explode("\n", $data['text'])))."\n</div>\n\n";
|
||||
$review = ReviewFormat::build($data);
|
||||
@ -24,7 +25,7 @@ final class ArticleArtifact
|
||||
$document = \App\Services\Ranking\ArticleDocument::enrich($document);
|
||||
}
|
||||
|
||||
return ['source_key' => $data['source_key'], 'revision' => $data['revision'], 'generation' => $data['generation'], 'article_code' => $code, 'relative_path' => 'jccp/'.$code.'.md', 'permalink' => $meta['permalink'], 'title' => $document['meta']['title'], 'format_version' => 'ttc-native-v1', 'markdown' => "---\n".Yaml::dump($document['meta'], 6, 2)."---\n".$document['body'], 'images' => $data['images']];
|
||||
return ['source_key' => $data['source_key'], 'revision' => $data['revision'], 'generation' => $data['generation'], 'article_code' => $code, 'relative_path' => 'jccp/'.$code.'.md', 'permalink' => $meta['permalink'], 'title' => $document['meta']['title'], 'format_version' => 'ttc-native-v2', 'markdown' => "---\n".Yaml::dump($document['meta'], 6, 2)."---\n".$document['body'], 'images' => $data['images']];
|
||||
}
|
||||
|
||||
public static function extension(string $mime): string
|
||||
|
||||
@ -6,6 +6,14 @@ use Illuminate\Http\Request;
|
||||
|
||||
final class ContentController
|
||||
{
|
||||
public function enrich(Request $request)
|
||||
{
|
||||
$data = $request->validate(['text' => 'present|string|max:100000']);
|
||||
$url = PackageTable::source($data['text']);
|
||||
|
||||
return response()->json(['environment' => config('content_center.environment'), 'package_table' => $url ? app(PackageTable::class)->fetch($url) : null]);
|
||||
}
|
||||
|
||||
public function preview(Request $request)
|
||||
{
|
||||
$artifact = app(ArticleArtifact::class)->build($request->all());
|
||||
|
||||
56
laravel/app/ContentCenter/LinkedText.php
Normal file
56
laravel/app/ContentCenter/LinkedText.php
Normal file
@ -0,0 +1,56 @@
|
||||
<?php
|
||||
|
||||
namespace App\ContentCenter;
|
||||
|
||||
final class LinkedText
|
||||
{
|
||||
public static function length(string $s): int
|
||||
{
|
||||
return intdiv(strlen(mb_convert_encoding($s, 'UTF-16LE', 'UTF-8')), 2);
|
||||
}
|
||||
|
||||
private static function escape(string $s): string
|
||||
{
|
||||
return htmlspecialchars($s, ENT_QUOTES | ENT_SUBSTITUTE, 'UTF-8');
|
||||
}
|
||||
|
||||
public static function safeUrl(string $s): bool
|
||||
{
|
||||
$u = parse_url($s);
|
||||
|
||||
return ! preg_match('/[\x00-\x20]/', $s) && $u && in_array($u['scheme'] ?? '', ['https', 'http'], true) && ! empty($u['host']) && ! isset($u['user']) && ! isset($u['pass']);
|
||||
}
|
||||
|
||||
private static function plain(string $s): string
|
||||
{
|
||||
return implode('', array_map(function ($v) {
|
||||
if (self::safeUrl($v)) {
|
||||
return '<a href="'.self::escape($v).'" target="_blank" rel="noopener noreferrer">'.self::escape($v).'</a>';
|
||||
}
|
||||
|
||||
return self::escape($v);
|
||||
}, preg_split('~(https?://[^\s<>\x22\x27]+)~u', $s, -1, PREG_SPLIT_DELIM_CAPTURE)));
|
||||
}
|
||||
|
||||
public static function line(string $line, int $base, array $links): string
|
||||
{
|
||||
$u = mb_convert_encoding($line, 'UTF-16LE', 'UTF-8');
|
||||
$n = intdiv(strlen($u), 2);
|
||||
$cursor = 0;
|
||||
$out = '';
|
||||
usort($links, fn ($a, $b) => $a['offset'] <=> $b['offset']);
|
||||
foreach ($links as $link) {
|
||||
$start = $link['offset'] - $base;
|
||||
$len = $link['length'];
|
||||
if ($start < 0 || $start < $cursor || $start + $len > $n || ! self::safeUrl($link['url'])) {
|
||||
continue;
|
||||
}
|
||||
$out .= self::plain(mb_convert_encoding(substr($u, $cursor * 2, ($start - $cursor) * 2), 'UTF-8', 'UTF-16LE'));
|
||||
$label = mb_convert_encoding(substr($u, $start * 2, $len * 2), 'UTF-8', 'UTF-16LE');
|
||||
$out .= '<a href="'.self::escape($link['url']).'" target="_blank" rel="noopener noreferrer">'.self::escape($label).'</a>';
|
||||
$cursor = $start + $len;
|
||||
}
|
||||
|
||||
return $out.self::plain(mb_convert_encoding(substr($u, $cursor * 2), 'UTF-8', 'UTF-16LE'));
|
||||
}
|
||||
}
|
||||
124
laravel/app/ContentCenter/PackageTable.php
Normal file
124
laravel/app/ContentCenter/PackageTable.php
Normal file
@ -0,0 +1,124 @@
|
||||
<?php
|
||||
|
||||
namespace App\ContentCenter;
|
||||
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use RuntimeException;
|
||||
|
||||
final class PackageTable
|
||||
{
|
||||
public static function source(string $text): ?string
|
||||
{
|
||||
if (preg_match('~论坛[：:]\s*(https?://(?:www\.)?ffqla\.net/thread-\d+-\d+-\d+\.html)~iu', $text, $m)) {
|
||||
return preg_replace('~^http:~i', 'https:', $m[1]);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static function allowed(string $url): bool
|
||||
{
|
||||
return preg_match('~^https://(?:www\.)?ffqla\.net/thread-\d+-\d+-\d+\.html$~D', $url) === 1;
|
||||
}
|
||||
|
||||
public function fetch(string $url): array
|
||||
{
|
||||
if (! self::allowed($url)) {
|
||||
throw new RuntimeException('套餐来源不受支持。');
|
||||
}
|
||||
|
||||
return Cache::remember('content-package-v2:'.hash('sha256', $url), 3600, function () use ($url) {
|
||||
for ($i = 0; $i < 3; $i++) {
|
||||
if (! self::allowed($url)) {
|
||||
throw new RuntimeException('套餐来源跳转不受支持。');
|
||||
}
|
||||
$r = Http::timeout(20)->connectTimeout(8)->withoutRedirecting()->withUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/115.0 Safari/537.36')->withOptions(['progress' => static function ($t, $n) {
|
||||
if ($t > 2097152 || $n > 2097152) {
|
||||
throw new RuntimeException('套餐响应过大。');
|
||||
}
|
||||
}])->get($url);
|
||||
if ($r->redirect()) {
|
||||
$next = $r->header('Location');
|
||||
if (str_starts_with($next, '/')) {
|
||||
$next = 'https://'.parse_url($url, PHP_URL_HOST).$next;
|
||||
}$url = $next;
|
||||
|
||||
continue;
|
||||
}
|
||||
if (! $r->successful() || strlen($r->body()) > 2097152) {
|
||||
throw new RuntimeException('套餐表暂不可用，请重新准备稿件。');
|
||||
}
|
||||
|
||||
return self::parse($r->body());
|
||||
}
|
||||
throw new RuntimeException('套餐来源跳转过多。');
|
||||
});
|
||||
}
|
||||
|
||||
public static function parse(string $html): array
|
||||
{
|
||||
$previous = libxml_use_internal_errors(true);
|
||||
$dom = new \DOMDocument;
|
||||
$dom->loadHTML('<?xml encoding="UTF-8">'.$html, LIBXML_NONET);
|
||||
libxml_clear_errors();
|
||||
libxml_use_internal_errors($previous);
|
||||
$xpath = new \DOMXPath($dom);
|
||||
$table = $xpath->query('//div[contains(concat(" ",normalize-space(@class)," ")," markdown-body ")]//table')->item(0);
|
||||
if (! $table) {
|
||||
throw new RuntimeException('来源没有套餐表，不能生成价格。');
|
||||
}
|
||||
foreach (iterator_to_array($xpath->query('.//script|.//style|.//iframe', $table)) as $node) {
|
||||
$node->parentNode->removeChild($node);
|
||||
}
|
||||
$headers = [];
|
||||
$rows = [];
|
||||
foreach ($xpath->query('./thead/tr|./tbody/tr|./tr', $table) as $tr) {
|
||||
$cells = [];
|
||||
$isHeader = false;
|
||||
foreach ($tr->childNodes as $td) {
|
||||
if (! in_array($td->nodeName, ['th', 'td'], true)) {
|
||||
continue;
|
||||
}$isHeader = $isHeader || $td->nodeName === 'th';
|
||||
$cells[] = trim(preg_replace('/\s+/u', ' ', $td->textContent));
|
||||
}
|
||||
if (! $cells) {
|
||||
continue;
|
||||
}if ($isHeader && ! $headers) {
|
||||
$headers = $cells;
|
||||
} else {
|
||||
$rows[] = $cells;
|
||||
}
|
||||
}
|
||||
$result = ['headers' => $headers, 'rows' => $rows];
|
||||
self::validate($result);
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
public static function validate(array $data): void
|
||||
{
|
||||
validator($data, ['headers' => 'required|array|size:7', 'headers.*' => 'required|string|max:100', 'rows' => 'required|array|min:1|max:100', 'rows.*' => 'array|size:7', 'rows.*.*' => 'present|string|max:1000'])->validate();
|
||||
if (! in_array('流量', $data['headers'], true) || ! in_array('月付', $data['headers'], true)) {
|
||||
throw new RuntimeException('套餐表头不匹配。');
|
||||
}
|
||||
}
|
||||
|
||||
public static function render(array $data): string
|
||||
{
|
||||
self::validate($data);
|
||||
$out = "<table v-pre>\n<thead>\n<tr>\n";
|
||||
foreach ($data['headers'] as $i => $s) {
|
||||
$out .= '<th style="text-align:'.($i === 0 ? 'left' : 'center').';">'.htmlspecialchars($s, ENT_QUOTES | ENT_SUBSTITUTE, 'UTF-8')."</th>\n";
|
||||
}
|
||||
$out .= "</tr>\n</thead>\n<tbody>\n";
|
||||
foreach ($data['rows'] as $row) {
|
||||
$out .= "<tr>\n";
|
||||
foreach ($row as $i => $s) {
|
||||
$out .= '<td style="text-align:'.($i === 0 ? 'left' : 'center').';">'.htmlspecialchars($s, ENT_QUOTES | ENT_SUBSTITUTE, 'UTF-8')."</td>\n";
|
||||
}$out .= "</tr>\n";
|
||||
}
|
||||
|
||||
return $out."</tbody>\n</table>\n";
|
||||
}
|
||||
}
|
||||
@ -18,18 +18,25 @@ final class ReviewFormat
|
||||
}
|
||||
$scores[] = $name.'：'.$m[1].' ';
|
||||
}
|
||||
$lines = explode("\n", trim($text));
|
||||
$lines = explode("\n", $text);
|
||||
$skipTitle = false;
|
||||
$title = $data['title'];
|
||||
if (preg_match('/^#([^\s#]+)/u', trim($lines[0]), $m)) {
|
||||
if (str_starts_with(trim($title), '#')) {
|
||||
$name = preg_replace('/机场$/u', '', $m[1]);
|
||||
$title = $name.'机场测评 & 测速';
|
||||
}
|
||||
array_shift($lines);
|
||||
$skipTitle = true;
|
||||
}
|
||||
$sections = ['intro' => [], 'navigation' => [], 'review' => [], 'extra' => []];
|
||||
$state = 'extra';
|
||||
foreach ($lines as $line) {
|
||||
$cursor = 0;
|
||||
foreach ($lines as $index => $line) {
|
||||
$offset = $cursor;
|
||||
$cursor += LinkedText::length($line) + 1;
|
||||
if ($index === 0 && $skipTitle) {
|
||||
continue;
|
||||
}
|
||||
$label = trim(preg_replace('/^[^\p{L}\p{N}]+/u', '', trim($line)));
|
||||
if (preg_match('/^主观评分/u', $label)) {
|
||||
$state = 'rating';
|
||||
@ -50,7 +57,8 @@ final class ReviewFormat
|
||||
$state = $key;
|
||||
$rest = trim(mb_substr($label, mb_strlen($heading)), " ：:\t");
|
||||
if ($rest !== '') {
|
||||
$sections[$key][] = $rest;
|
||||
$prefix = mb_strpos($line, $rest);
|
||||
$sections[$key][] = ['text' => $rest, 'offset' => $offset + LinkedText::length(mb_substr($line, 0, $prefix))];
|
||||
}
|
||||
$matched = true;
|
||||
break;
|
||||
@ -59,29 +67,38 @@ final class ReviewFormat
|
||||
if ($matched || in_array($state, ['rating', 'reminder'], true)) {
|
||||
continue;
|
||||
}
|
||||
$sections[$state][] = $line;
|
||||
if ($state === 'navigation' && preg_match('/^(?:论坛|套餐)[：:]/u', $label)) {
|
||||
continue;
|
||||
}
|
||||
$sections[$state][] = ['text' => $line, 'offset' => $offset];
|
||||
}
|
||||
$body = "::: tip 天天测提醒您，任何机场都有跑路可能，建议仅月付，以降低风险。（不针对任何机场） \n::: \n\n";
|
||||
$body .= "::: important 天天测打分 \n".implode("\n", $scores)."\n::: \n\n";
|
||||
if (trim(implode('', $sections['review'])) !== '') {
|
||||
$body .= "::: important 天天测点评 \n".self::paragraphs($sections['review'])."\n::: \n\n";
|
||||
if (trim(implode('', array_column($sections['review'], 'text'))) !== '') {
|
||||
$body .= "::: important 天天测点评 \n".self::paragraphs($sections['review'], $data['links'] ?? [])."\n::: \n\n";
|
||||
}
|
||||
foreach (['intro' => '机场自介', 'navigation' => '链接导航'] as $key => $heading) {
|
||||
if (trim(implode('', $sections[$key])) !== '') {
|
||||
$body .= '## '.$heading." \n\n".self::paragraphs($sections[$key])."\n\n";
|
||||
if (trim(implode('', array_column($sections[$key], 'text'))) !== '') {
|
||||
$body .= '## '.$heading." \n\n".self::paragraphs($sections[$key], $data['links'] ?? [])."\n\n";
|
||||
}
|
||||
}
|
||||
if (trim(implode('', $sections['extra'])) !== '') {
|
||||
$body .= self::paragraphs($sections['extra'])."\n\n";
|
||||
if (trim(implode('', array_column($sections['extra'], 'text'))) !== '') {
|
||||
$body .= self::paragraphs($sections['extra'], $data['links'] ?? [])."\n\n";
|
||||
}
|
||||
if (PackageTable::source($data['text']) && empty($data['package_table'])) {
|
||||
throw new \RuntimeException('套餐尚未提取，请重新准备稿件。');
|
||||
}
|
||||
if (! empty($data['package_table'])) {
|
||||
$body .= "## 套餐价格 \n\n".PackageTable::render($data['package_table'])."\n";
|
||||
}
|
||||
$body .= "## 节点测试 \n\n";
|
||||
|
||||
return ['title' => $title, 'tags' => ['机场测评', '每天测速'], 'body' => $body];
|
||||
}
|
||||
|
||||
private static function paragraphs(array $lines): string
|
||||
private static function paragraphs(array $lines, array $links): string
|
||||
{
|
||||
// Keep channel text inert while using exactly the site's native containers/headings.
|
||||
return '<div v-pre>'."\n".implode("\n", array_map(fn ($line) => trim($line) === '' ? '' : '<p>'.htmlspecialchars($line, ENT_QUOTES | ENT_SUBSTITUTE, 'UTF-8').'</p>', $lines))."\n</div>";
|
||||
return '<div v-pre>'."\n".implode("\n", array_map(fn ($line) => trim($line['text']) === '' ? '' : '<p>'.LinkedText::line($line['text'], $line['offset'], $links).'</p>', $lines))."\n</div>";
|
||||
}
|
||||
}
|
||||
|
||||
@ -5,6 +5,7 @@ use App\ContentCenter\ContentController;
|
||||
use Illuminate\Support\Facades\Route;
|
||||
|
||||
Route::prefix('api/content/v1')->middleware([ContentAuthenticate::class.':atom', 'throttle:30,1'])->group(function () {
|
||||
Route::post('/enrich', [ContentController::class, 'enrich']);
|
||||
Route::post('/delete', [ContentController::class, 'delete']);
|
||||
Route::post('/preview', [ContentController::class, 'preview']);
|
||||
Route::post('/publish', [ContentController::class, 'publish']);
|
||||
|
||||
@ -216,4 +216,63 @@ final class ContentCenterTest extends TestCase
|
||||
$this->signed('/api/content/v1/delete', $data)->assertOk();
|
||||
$this->assertFileDoesNotExist($this->root.'/vuepress/docs/'.$artifact['relative_path']);
|
||||
}
|
||||
|
||||
private function packageFixture(): array
|
||||
{
|
||||
return ['headers' => ['套餐', '流量', '月付', '季付', '半年', '年付', '一次性'], 'rows' => [['基础', '100GB', '10.00', '×', '×', '×', '×']]];
|
||||
}
|
||||
|
||||
public function test_hidden_official_link_and_separate_frozen_package_table_follow_legacy_layout(): void
|
||||
{
|
||||
$input = $this->reviewInput();
|
||||
$input['text'] = str_replace('官网：点击进入', "官网：点击进入\n套餐：https://example.com/package\n论坛：https://www.ffqla.net/thread-932-1-1.html", $input['text']);
|
||||
$offset = \App\ContentCenter\LinkedText::length(substr($input['text'], 0, strpos($input['text'], '点击进入')));
|
||||
$input['links'] = [['offset' => $offset, 'length' => 4, 'url' => 'https://example.com/shop?a=1&b=2']];
|
||||
$input['package_table'] = $this->packageFixture();
|
||||
$a = app(ArticleArtifact::class)->build($input);
|
||||
$md = $a['markdown'];
|
||||
$this->assertStringContainsString('href="https://example.com/shop?a=1&b=2"', $md);
|
||||
$this->assertStringContainsString('>点击进入</a>', $md);
|
||||
$this->assertStringContainsString('## 套餐价格', $md);
|
||||
$this->assertStringContainsString('100GB', $md);
|
||||
$nav = explode('## 套餐价格', explode('## 链接导航', $md)[1])[0];
|
||||
$this->assertStringNotContainsString('论坛', $nav);
|
||||
$this->assertStringNotContainsString('套餐', $nav);
|
||||
Http::assertNothingSent();
|
||||
$input['package_table']['rows'][0][2] = '12.00';
|
||||
$this->assertNotSame(ArticleArtifact::hash($a), ArticleArtifact::hash(app(ArticleArtifact::class)->build($input)));
|
||||
}
|
||||
|
||||
public function test_package_parser_uses_first_markdown_table_and_discards_active_markup(): void
|
||||
{
|
||||
$html = '<div class="markdown-body"><table><thead><tr>';
|
||||
foreach ($this->packageFixture()['headers'] as $v) {
|
||||
$html .= '<th>'.$v.'</th>';
|
||||
}
|
||||
$html .= '</tr></thead><tbody><tr>';
|
||||
foreach ($this->packageFixture()['rows'][0] as $v) {
|
||||
$html .= '<td onclick="bad()">'.$v.'<script>bad()</script></td>';
|
||||
}
|
||||
$html .= '</tr></tbody></table><table><tr><td>ignore</td></tr></table></div>';
|
||||
$t = \App\ContentCenter\PackageTable::parse($html);
|
||||
$this->assertSame($this->packageFixture(), $t);
|
||||
$rendered = \App\ContentCenter\PackageTable::render($t);
|
||||
$this->assertStringNotContainsString('script', $rendered);
|
||||
$this->assertStringNotContainsString('onclick', $rendered);
|
||||
}
|
||||
|
||||
public function test_missing_package_data_cannot_silently_publish_or_fetch_during_build(): void
|
||||
{
|
||||
$input = $this->reviewInput();
|
||||
$input['text'] .= "\n论坛：https://www.ffqla.net/thread-932-1-1.html";
|
||||
$this->expectException(\RuntimeException::class);
|
||||
app(ArticleArtifact::class)->build($input);
|
||||
}
|
||||
|
||||
public function test_package_fetch_rejects_redirect_to_unregistered_host(): void
|
||||
{
|
||||
Http::fake(['https://www.ffqla.net/*' => Http::response('', 302, ['Location' => 'http://127.0.0.1/private'])]);
|
||||
$this->expectException(\RuntimeException::class);
|
||||
app(\App\ContentCenter\PackageTable::class)->fetch('https://www.ffqla.net/thread-932-1-1.html');
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user