Skip stale freenode sources with durable TeleHub warnings

app/Services/Freenode/Pipeline.php

@@ -16,24 +16,33 @@ final class Pipeline
         return new ReleaseStore(storage_path('app/freenode-v2'), public_path('freenode/merge'));
     }
 
+    public static function warnings(): SourceWarnings
+    {
+        return new SourceWarnings(storage_path('app/freenode-telehub'),
+            (string) config('freenode.telehub_base_url'), (string) config('freenode.telehub_api_key'));
+    }
+
     public static function prepare(): array
     {
         self::store()->assertWriter();
-        $sources = [];
+        try { self::warnings()->flush(); }
+        catch (\Throwable) { syslog(LOG_WARNING, 'master01 freenode_warning_outbox_unavailable'); }
         $cycle = strtotime(date('Y-m-d').' '.sprintf('%02d', intdiv((int) date('H'), 6) * 6).':00:00');
-        foreach (CrawlFreenode::where('status', 1)->get() as $source) {
-            if (!preg_match('/^[a-zA-Z0-9_-]+$/D', $source->name)) throw new RuntimeException('invalid_source_name');
-            $row = ['name' => $source->name];
-            foreach (json_decode($source->file_real ?? '[]', true, 512, JSON_THROW_ON_ERROR) as $format => $file) {
-                if (!in_array($format, ['v2ray', 'clash'], true)) continue;
-                if (!preg_match('/^'.date('Ymd').'-(v2ray\.txt|clash\.yaml)$/D', $file)) throw new RuntimeException('stale_source_date:'.$source->name);
-                $path = public_path('freenode/from/'.$source->name.'/'.$file);
-                if (!is_file($path) || filemtime($path) < $cycle) throw new RuntimeException('stale_source_cycle:'.$source->name);
-                $row[$format] = file_get_contents($path);
+        $sources = SourceSelector::collect(
+            CrawlFreenode::where('status', 1)->get()->toArray(),
+            public_path('freenode/from'), date('Ymd'), $cycle,
+            function (string $source, string $reason): void {
+                try {
+                    $receipt = self::warnings()->notify($source, $reason, date('Ymd'));
+                    \Illuminate\Support\Facades\Log::warning('freenode_source_skipped', $receipt);
+                } catch (\Throwable) {
+                    // A notification failure must not block remaining valid sources.
+                    syslog(LOG_WARNING, 'master01 freenode_source_skipped '.json_encode([
+                        'source' => $source, 'reason' => $reason, 'notification' => 'outbox_failed',
+                    ]));
+                }
             }
-            if (!isset($row['v2ray'], $row['clash'])) throw new RuntimeException('incomplete_source:'.$source->name);
-            $sources[] = $row;
-        }
+        );
         $sites = [];
         foreach (SiteMap::whereIn('group_code', ['fn_a', 'fn_b'])->where('status', 1)->get() as $site) {
             $config = json_decode($site->config ?? '[]', true, 512, JSON_THROW_ON_ERROR);

app/Services/Freenode/SourceSelector.php

@@ -0,0 +1,48 @@
+<?php
+
+declare(strict_types=1);
+namespace App\Services\Freenode;
+
+use RuntimeException;
+
+final class SourceSelector
+{
+    /** Never mix a fresh format with a stale format from the same source. */
+    public static function collect(iterable $sources, string $directory, string $date, int $cycle, callable $warning): array
+    {
+        $ready = [];
+        foreach ($sources as $source) {
+            $name = $source['name'];
+            if (!preg_match('/^[a-zA-Z0-9_-]+$/D', $name)) throw new RuntimeException('invalid_source_name');
+            $files = json_decode($source['file_real'] ?? '[]', true, 512, JSON_THROW_ON_ERROR);
+            $paths = [];
+            $reason = null;
+            foreach ($files as $format => $file) {
+                if (!in_array($format, ['v2ray', 'clash'], true)) continue;
+                if (!is_string($file) || !preg_match('/^'.preg_quote($date, '/').'-(v2ray\.txt|clash\.yaml)$/D', $file)) {
+                    $reason = 'stale_source_date';
+                    break;
+                }
+                $path = $directory.'/'.$name.'/'.$file;
+                if (!is_file($path) || filemtime($path) < $cycle) {
+                    $reason = 'stale_source_cycle';
+                    break;
+                }
+                $paths[$format] = $path;
+            }
+            if ($reason !== null) {
+                $warning($name, $reason);
+                continue;
+            }
+            if (!isset($paths['v2ray'], $paths['clash'])) throw new RuntimeException('incomplete_source:'.$name);
+            $row = ['name' => $name];
+            foreach ($paths as $format => $path) {
+                $content = file_get_contents($path);
+                if ($content === false) throw new RuntimeException('source_read_failed:'.$name);
+                $row[$format] = $content;
+            }
+            $ready[] = $row;
+        }
+        return $ready;
+    }
+}

app/Services/Freenode/SourceWarnings.php

@@ -0,0 +1,128 @@
+<?php
+
+declare(strict_types=1);
+namespace App\Services\Freenode;
+
+use Closure;
+use Illuminate\Support\Facades\Http;
+use RuntimeException;
+
+/** Durable, metadata-only TeleHub warning outbox. No Telegram credentials here. */
+final class SourceWarnings
+{
+    private ?Closure $transport;
+
+    public function __construct(private string $directory, private string $baseUrl, private string $apiKey, ?callable $transport = null)
+    {
+        $this->transport = $transport === null ? null : Closure::fromCallable($transport);
+    }
+
+    public function notify(string $source, string $reason, string $date): array
+    {
+        if (!preg_match('/^[a-zA-Z0-9_-]+$/D', $source) || !preg_match('/^\d{8}$/D', $date)
+            || !in_array($reason, ['stale_source_date', 'stale_source_cycle'], true)) throw new RuntimeException('invalid_source_warning');
+        $key = 'master01.production.freenode.source-stale.'.$date.'.'.$source;
+        return $this->locked(function () use ($source, $reason, $date, $key) {
+            $path = $this->directory.'/'.hash('sha256', $key).'.json';
+            if (is_file($path)) $event = json_decode(file_get_contents($path), true, 512, JSON_THROW_ON_ERROR);
+            else {
+                $event = [
+                    'key' => $key, 'source' => $source, 'date' => $date, 'reason' => $reason,
+                    'payload' => ['bot' => 'warning', 'target' => 'base', 'text' =>
+                        "[master01][production] 免费订阅来源过期\n日期：".$date."\n来源：".$source.
+                        "\n原因：".($reason === 'stale_source_date' ? '未取得当天文件' : '本轮文件缺失或未更新').
+                        "\n处理：已跳过该来源，继续使用其他有效来源生成；无有效来源或校验失败时保留上一批次。"],
+                    'state' => 'pending', 'receipt' => null,
+                ];
+                // Persist exact body before the first HTTP attempt, for stable retries.
+                $this->save($path, $event);
+            }
+            $event = $this->deliver($event);
+            $this->save($path, $event);
+            return $this->summary($event);
+        });
+    }
+
+    public function flush(): array
+    {
+        return $this->locked(function () {
+            $results = [];
+            foreach (array_slice(glob($this->directory.'/*.json') ?: [], -50) as $path) {
+                $event = json_decode(file_get_contents($path), true, 512, JSON_THROW_ON_ERROR);
+                $event = $this->deliver($event);
+                $this->save($path, $event);
+                $results[] = $this->summary($event);
+            }
+            return $results;
+        });
+    }
+
+    private function deliver(array $event): array
+    {
+        if (in_array($event['state'], ['sent', 'failed', 'uncertain', 'cancelled', 'rejected'], true)) return $event;
+        if (!$this->apiKey || !preg_match('~^https://[^/]+(?:/[^?#]*)?$~D', $this->baseUrl)) {
+            $event['error'] = 'telehub_not_configured';
+            return $event;
+        }
+        try {
+            $method = $event['receipt'] ? 'GET' : 'POST';
+            $path = '/api/v1/messages'.($event['receipt'] ? '/'.$event['receipt'] : '');
+            $response = $this->request($method, $path, $event);
+            if (!in_array($response['status'], [200, 202], true)) {
+                $event['error'] = 'telehub_http_'.$response['status'];
+                if ($response['status'] >= 400 && $response['status'] < 500 && $response['status'] !== 429) $event['state'] = 'rejected';
+                return $event;
+            }
+            $data = $response['data']['data'] ?? [];
+            if (!is_string($data['id'] ?? null) || !preg_match('/^[a-f0-9-]{36}$/D', $data['id'])
+                || !in_array($data['status'] ?? '', ['queued', 'sending', 'sent', 'failed', 'uncertain', 'cancelled'], true)) {
+                $event['error'] = 'telehub_invalid_receipt';
+                return $event;
+            }
+            $event['receipt'] = $data['id'];
+            $event['state'] = $data['status'];
+            $event['error'] = $data['error_code'] ?? null;
+        } catch (\Throwable) {
+            // Retrying an ambiguous POST uses the same frozen payload and key.
+            $event['error'] = 'telehub_request_failed';
+        }
+        return $event;
+    }
+
+    private function request(string $method, string $path, array $event): array
+    {
+        if ($this->transport) return ($this->transport)($method, $path, $event['payload'], $event['key']);
+        $http = Http::withToken($this->apiKey)->acceptJson()->connectTimeout(5)->timeout(10)->withoutRedirecting();
+        $response = $method === 'POST'
+            ? $http->withHeaders(['Idempotency-Key' => $event['key']])->post(rtrim($this->baseUrl, '/').$path, $event['payload'])
+            : $http->get(rtrim($this->baseUrl, '/').$path);
+        return ['status' => $response->status(), 'data' => $response->json()];
+    }
+
+    private function summary(array $event): array
+    {
+        return array_intersect_key($event, array_flip(['key', 'source', 'date', 'reason', 'state', 'receipt', 'error']));
+    }
+
+    private function save(string $path, array $event): void
+    {
+        $tmp = tempnam($this->directory, '.warning-');
+        try {
+            if (file_put_contents($tmp, json_encode($event, JSON_THROW_ON_ERROR | JSON_UNESCAPED_UNICODE)) === false) throw new RuntimeException('warning_write_failed');
+            chmod($tmp, 0600);
+            if (!rename($tmp, $path)) throw new RuntimeException('warning_rename_failed');
+        } finally { if (is_file($tmp)) unlink($tmp); }
+    }
+
+    private function locked(callable $work): mixed
+    {
+        if (!is_dir($this->directory) && !mkdir($this->directory, 0700, true) && !is_dir($this->directory)) throw new RuntimeException('warning_directory_failed');
+        $lock = fopen($this->directory.'/outbox.lock', 'c');
+        if (!$lock) throw new RuntimeException('warning_lock_failed');
+        chmod($this->directory.'/outbox.lock', 0600);
+        try {
+            if (!flock($lock, LOCK_EX)) throw new RuntimeException('warning_lock_failed');
+            return $work();
+        } finally { flock($lock, LOCK_UN); fclose($lock); }
+    }
+}

config/freenode.php

@@ -2,6 +2,9 @@
 
 return [
 
+    'telehub_base_url' => env('TELEHUB_BASE_URL', ''),
+    'telehub_api_key' => env('TELEHUB_API_KEY', ''),
+
     'engine' => env('FREENODE_ENGINE', 'legacy'),
     'singbox_binary' => env('FREENODE_SINGBOX_BINARY', '/opt/master01-freenode/bin/sing-box'),
     'mihomo_binary' => env('FREENODE_MIHOMO_BINARY', '/opt/master01-freenode/bin/mihomo'),

docs/maintenance/FREENODE.md

@@ -272,3 +272,11 @@ SubCheck正式复检任务`28ca01e368dd4255843646407b95927c`于北京时间2026-
 功能提交`2b50ac3670ce240b9f7aafae5e00fffa7c40bed2`已推送并快进部署；生产root回归27项64断言全部通过，www-data回归27项61断言、仅root专属项按设计跳过。额外实际以root调用publish，确认返回身份不符且current SHA未变；www-data读取今日16份完整产物及文章计数1855均通过。私有发布树内没有root属主文件，所有原并行Git状态保留。
 
 本轮12:12自动发布曾再次因相同权限失败，修复后以www-data补发布今日候选成功，两份清单继续www-data:0600；12:15:02—12:15:12原有定时分发自行完成，无需重放全部cron。四站10份今日落盘与本批次哈希匹配（Clash保持既有trim差异），Nodehub服务器侧10个精确HTTPS地址均200、响应哈希与文件一致。已完成权限修复、防复发部署与服务器侧验收，未做线上页面验收或节点联网测试。代码回退点`676196388df7c302e5cbfa14cbdb54f364cdccd8`；属主恢复与代码回退分别处理，不应为回退代码把清单再改回root。
+
+## 过期来源降级与TeleHub warning（2026-10-28）
+
+用户明确要求：某一来源过期时通过TeleHub warning告警并跳过，不终止整批生成。SourceSelector按来源完整校验当天日期和当前六小时周期；任一格式过期/缺失即跳过整个来源，避免混用新旧格式。其他有效来源继续走原Builder、固定核心、非空格式、完整性与原子发布流程。所有来源不可用时仍失败并保留旧批次。此前数量骤减门禁已按用户历史要求移除，本次不新增或恢复数量比例限制。
+
+SourceWarnings使用独立master01生产调用方，仅bot=warning、target=base。配置TELEHUB_BASE_URL/TELEHUB_API_KEY经config/freenode.php读取；凭据仅部署在服务器私密环境，不存机器人Token/chat_id。storage/app/freenode-telehub为0700，事件和锁0600，按来源+北京时间日期持久幂等键和冻结纯文本；同日重复过期不重复新建消息。网络结果不明重试同键同内容；accepted/queued只查询原UUID，不把接收当作sent；failed/uncertain等终态不盲目补发，不降级改发其他机器人。每次prepare尝试处理未完成回执，亦可用tools/freenode/manage.php warnings核对。通知失败保留脱敏状态并继续有效来源生成；异常堆栈、订阅内容、凭据不入告警。
+
+回退需将代码恢复至本次前提交，并按需撤销专用调用方，不覆盖业务数据库、已发布文章或其他项目配置；订阅批次回退仍使用原ReleaseStore流程。代码/生产部署与实测状态见本节后续验收记录。

tests/Unit/Freenode/SourceSelectorTest.php

@@ -0,0 +1,52 @@
+<?php
+namespace Tests\Unit\Freenode;
+use App\Services\Freenode\SourceSelector;
+use PHPUnit\Framework\TestCase;
+
+final class SourceSelectorTest extends TestCase
+{
+    private string $dir;
+    protected function setUp(): void { $this->dir = sys_get_temp_dir().'/fn-select-'.bin2hex(random_bytes(5)); mkdir($this->dir); }
+    protected function tearDown(): void {
+        foreach (glob($this->dir.'/*/*') ?: [] as $p) unlink($p);
+        foreach (glob($this->dir.'/*') ?: [] as $p) rmdir($p);
+        rmdir($this->dir);
+    }
+    private function source(string $name, string $date, int $mtime): array {
+        mkdir($this->dir.'/'.$name);
+        $files = ['v2ray' => $date.'-v2ray.txt', 'clash' => $date.'-clash.yaml'];
+        foreach ($files as $file) { file_put_contents($this->dir.'/'.$name.'/'.$file, $name); touch($this->dir.'/'.$name.'/'.$file, $mtime); }
+        return ['name' => $name, 'file_real' => json_encode($files)];
+    }
+    public function testStaleDateWarnsOnceAndKeepsFreshSource(): void {
+        $old = $this->source('old', '20261027', 100);
+        $fresh = $this->source('fresh', '20261028', 200);
+        $warnings = [];
+        $result = SourceSelector::collect([$old, $fresh], $this->dir, '20261028', 150, function (...$event) use (&$warnings) { $warnings[] = $event; });
+        self::assertSame([['old', 'stale_source_date']], $warnings);
+        self::assertSame([['name' => 'fresh', 'v2ray' => 'fresh', 'clash' => 'fresh']], $result);
+    }
+    public function testOneStaleFormatSkipsEntireSourceAndContinues(): void {
+        $old = $this->source('old', '20261028', 200);
+        touch($this->dir.'/old/20261028-clash.yaml', 100);
+        $fresh = $this->source('fresh', '20261028', 200);
+        $warnings = [];
+        $result = SourceSelector::collect([$old, $fresh], $this->dir, '20261028', 150, function (...$event) use (&$warnings) { $warnings[] = $event; });
+        self::assertSame([['old', 'stale_source_cycle']], $warnings);
+        self::assertSame('fresh', $result[0]['name']);
+    }
+    public function testAllStaleReturnsNoSourcesWithoutReusingOldContent(): void {
+        $old = $this->source('old', '20261027', 100);
+        $warnings = [];
+        self::assertSame([], SourceSelector::collect([$old], $this->dir, '20261028', 150, function (...$event) use (&$warnings) { $warnings[] = $event; }));
+        self::assertCount(1, $warnings);
+    }
+    public function testInvalidMetadataIsNotSilentlyDowngraded(): void {
+        $this->expectException(\RuntimeException::class);
+        SourceSelector::collect([['name' => '../escape', 'file_real' => '{}']], $this->dir, '20261028', 150, fn () => null);
+    }
+    public function testIncompleteFreshSourceStillFails(): void {
+        $this->expectExceptionMessage('incomplete_source:empty');
+        SourceSelector::collect([['name' => 'empty', 'file_real' => '{}']], $this->dir, '20261028', 150, fn () => null);
+    }
+}

tests/Unit/Freenode/SourceWarningsTest.php

@@ -0,0 +1,58 @@
+<?php
+namespace Tests\Unit\Freenode;
+use App\Services\Freenode\SourceWarnings;
+use PHPUnit\Framework\TestCase;
+
+final class SourceWarningsTest extends TestCase
+{
+    private string $dir;
+    private string $id = '12345678-1234-1234-1234-123456789abc';
+    protected function setUp(): void { $this->dir = sys_get_temp_dir().'/fn-warning-'.bin2hex(random_bytes(5)); }
+    protected function tearDown(): void { foreach (glob($this->dir.'/*') ?: [] as $p) unlink($p); if (is_dir($this->dir)) rmdir($this->dir); }
+    public function testRoutesWarningToBaseAndDailyDuplicateOnlyQueriesReceipt(): void {
+        $calls = [];
+        $transport = function ($method, $path, $payload, $key) use (&$calls) {
+            $calls[] = [$method, $path, $payload, $key];
+            return ['status' => $method === 'POST' ? 202 : 200, 'data' => ['data' => ['id' => $this->id, 'status' => $method === 'POST' ? 'queued' : 'sent']]];
+        };
+        $outbox = new SourceWarnings($this->dir, 'https://telehub.example', 'fake-secret', $transport);
+        self::assertSame('queued', $outbox->notify('old', 'stale_source_date', '20261028')['state']);
+        self::assertSame('sent', $outbox->notify('old', 'stale_source_cycle', '20261028')['state']);
+        $outbox->notify('old', 'stale_source_date', '20261028');
+        self::assertCount(2, $calls);
+        self::assertSame(['POST', 'GET'], array_column($calls, 0));
+        self::assertSame('warning', $calls[0][2]['bot']);
+        self::assertSame('base', $calls[0][2]['target']);
+        self::assertStringContainsString('已跳过该来源', $calls[0][2]['text']);
+        self::assertStringContainsString('old', $calls[0][2]['text']);
+        self::assertSame(0600, fileperms(glob($this->dir.'/*.json')[0]) & 0777);
+        self::assertStringNotContainsString('fake-secret', file_get_contents(glob($this->dir.'/*.json')[0]));
+    }
+    public function testAmbiguousRequestRetriesSameBodyAndKeyAfterReconstruction(): void {
+        $calls = [];
+        $transport = function ($method, $path, $payload, $key) use (&$calls) {
+            $calls[] = [$payload, $key];
+            if (count($calls) === 1) throw new \RuntimeException('private HTTP exception');
+            return ['status' => 200, 'data' => ['data' => ['id' => $this->id, 'status' => 'sent']]];
+        };
+        $first = new SourceWarnings($this->dir, 'https://telehub.example', 'fake-secret', $transport);
+        self::assertSame('telehub_request_failed', $first->notify('old', 'stale_source_date', '20261028')['error']);
+        $second = new SourceWarnings($this->dir, 'https://telehub.example', 'fake-secret', $transport);
+        self::assertSame('sent', $second->flush()[0]['state']);
+        self::assertSame($calls[0], $calls[1]);
+        self::assertStringNotContainsString('private HTTP', file_get_contents(glob($this->dir.'/*.json')[0]));
+    }
+    public function testExplicitlyRejectedWarningNeverFallsBackToAnotherBot(): void {
+        $calls = 0;
+        $transport = function ($method, $path, $payload) use (&$calls) { $calls++; self::assertSame('warning', $payload['bot']); return ['status' => 403, 'data' => []]; };
+        $outbox = new SourceWarnings($this->dir, 'https://telehub.example', 'fake-secret', $transport);
+        self::assertSame('rejected', $outbox->notify('old', 'stale_source_date', '20261028')['state']);
+        $outbox->flush();
+        self::assertSame(1, $calls);
+    }
+    public function testMissingConfigurationPersistsPendingEventWithoutHTTP(): void {
+        $outbox = new SourceWarnings($this->dir, '', '', fn () => self::fail('Must not send'));
+        self::assertSame('telehub_not_configured', $outbox->notify('old', 'stale_source_date', '20261028')['error']);
+        self::assertCount(1, glob($this->dir.'/*.json'));
+    }
+}

tools/freenode/manage.php

@@ -9,6 +9,7 @@ try {
     $action = $argv[1] ?? 'status';
     $store = Pipeline::store();
     $result = match ($action) {
+        'warnings' => Pipeline::warnings()->flush(),
         'prepare' => Pipeline::prepare(),
         'publish' => $store->publishPrepared(),
         'rollback' => $store->activate($argv[2] ?? ($store->previous()['id'] ?? 'missing')),