From 67f1c93e02cff127ae74c16e886212da85832564 Mon Sep 17 00:00:00 2001 From: hellcat Date: Tue, 13 Oct 2026 19:53:10 +0800 Subject: [PATCH] Add isolated gateway admin and white proxy provisioning --- docs/maintenance/GATEWAY_ADMIN_PROXY.md | 56 +++++ scripts/gateway-admin-proxy/.gitignore | 2 + scripts/gateway-admin-proxy/deploy.py | 149 +++++++++++++ .../gateway-admin-cert.service | 5 + .../gateway-admin-cert.timer | 8 + .../install-origin-cert.py | 34 +++ .../issue-gateway-admin.sh | 17 ++ .../gateway-admin-proxy/issue-white-legacy.sh | 17 ++ .../gateway-admin-proxy/legacy-dns-hook.py | 116 ++++++++++ scripts/gateway-admin-proxy/render.py | 200 ++++++++++++++++++ .../renew-gateway-admin.sh | 9 + .../gateway-admin-proxy/renew-white-legacy.sh | 9 + scripts/gateway-admin-proxy/sync-cf-ips.py | 45 ++++ scripts/gateway-admin-proxy/verify-maps.py | 85 ++++++++ .../white-proxy-cert.service | 5 + .../white-proxy-cert.timer | 8 + .../white-proxy-cloudflare-ips.service | 5 + .../white-proxy-cloudflare-ips.timer | 8 + 18 files changed, 778 insertions(+) create mode 100644 docs/maintenance/GATEWAY_ADMIN_PROXY.md create mode 100644 scripts/gateway-admin-proxy/.gitignore create mode 100644 scripts/gateway-admin-proxy/deploy.py create mode 100644 scripts/gateway-admin-proxy/gateway-admin-cert.service create mode 100644 scripts/gateway-admin-proxy/gateway-admin-cert.timer create mode 100644 scripts/gateway-admin-proxy/install-origin-cert.py create mode 100644 scripts/gateway-admin-proxy/issue-gateway-admin.sh create mode 100644 scripts/gateway-admin-proxy/issue-white-legacy.sh create mode 100644 scripts/gateway-admin-proxy/legacy-dns-hook.py create mode 100644 scripts/gateway-admin-proxy/render.py create mode 100644 scripts/gateway-admin-proxy/renew-gateway-admin.sh create mode 100644 scripts/gateway-admin-proxy/renew-white-legacy.sh create mode 100644 scripts/gateway-admin-proxy/sync-cf-ips.py create mode 100644 scripts/gateway-admin-proxy/verify-maps.py create mode 100644 scripts/gateway-admin-proxy/white-proxy-cert.service create mode 100644 scripts/gateway-admin-proxy/white-proxy-cert.timer create mode 100644 scripts/gateway-admin-proxy/white-proxy-cloudflare-ips.service create mode 100644 scripts/gateway-admin-proxy/white-proxy-cloudflare-ips.timer diff --git a/docs/maintenance/GATEWAY_ADMIN_PROXY.md b/docs/maintenance/GATEWAY_ADMIN_PROXY.md new file mode 100644 index 0000000..eca1820 --- /dev/null +++ b/docs/maintenance/GATEWAY_ADMIN_PROXY.md @@ -0,0 +1,56 @@ +# 网关管理域名与白色机反代 + +用户指定日期：2026-10-13。范围为两套既有网关管理容器的新域名，以及白色机的新版/旧版域名区域映射和两机场主站入口。源码与配置生成工具在 `scripts/gateway-admin-proxy/`；跨项目域名映射的唯一登记正文在 CoDocs `20-基础设施/访问与反代.md`。 + +## 当前阶段：配置准备，未完成上线验收 + +- 两条新管理 A 记录已在 Namecheap 保存：getway-fuc.gemcook.net、getway-dpl.gemcook.net 指向 gateway-01；原记录和 NS 保留。 +- 新 CF 账户可访问。tomshell.online、tomtojapan.online 的根 A 与泛域 CNAME 均指向白色机、橙云，本轮已读取页面确认。 +- 两个新区域的 Cloudflare Origin CA 证书均已签发，覆盖各自根域和一级泛域，有效至 2041-10-09。私钥在白色机独立目录生成，没有发送到 CF 或写入 Git。证书安装与 Full (strict) 复验尚未完成。 +- 映射/请求头逻辑通过 81 项真实 Nginx 的回环集成检查，没有访问线上源站。服务器部署、后台登录和旧入口回归尚未完成。 +- 旧 tomshell.com.de 的根证书已过期，旧泛域匹配用的是仅覆盖 umami 的证书。Hostinger 已可访问；用户批准的 `_acme-challenge` CNAME 已保存并经权威 DNS 确认，指向 `old-com-de.acme.tomshell.online`。限定一个区域、白色机 IPv4 的 CF DNS 编辑令牌尚未成功创建；不关闭证书校验替代。 + +后续每一阶段按实际结果替换本节，不把配置生成、DNS 保存或证书签发当成业务上线完成。 + +## 用户指定的边界 + +| 用途 | 源站标识 | 白色机入口标识 | +| --- | --- | --- | +| fuc 网关管理 | getway-fuc.gemcook.net | getway-fuc.tomshell.online | +| dpl 网关管理 | getway-dpl.gemcook.net | getway-dpl.tomshell.online | +| 新版域名区域 | gemcook.net；同前缀一级子域 | tomshell.online；同前缀一级子域 | +| 旧版域名区域 | diaodu.us；同前缀一级子域 | tomshell.com.de；同前缀一级子域 | +| dpl 主站 | dp-link.net | dpl.tomtojapan.online | +| fuc 主站 | 福云.com（ASCII：xn--9kqy16h.com） | fuc.tomtojapan.online | + +主域名映射不会自行完成子域 DNS、证书或 Nginx 匹配。本次泛映射按同一前缀对应源站，拒绝不属于指定区域和多级子域的转发；多级主机需额外证书和登记。现有专用匹配优先保留作为兼容入口，不能把已有 TTC、Git、日语服务或旧网关入口送往错误上游。 + +gemcook.net 根域本次 DNS 查询没有 A/AAAA，根反代配置不能据此声称有可访问的根站点。泛映射也不证明任意源站子域已经部署；逐实际入口验收。 + +本地访问只经登记的白色机入口，包含资源、重定向、API 和 WebSocket。没有通过源站 IP、Host 覆盖、hosts 或另一机器代抓取补本地验证。 + +## 网关与共享资源 + +两个现有管理容器保持独立，不改账号、密码、订阅上游、卷或原业务监听端口。gateway-01 新增精确管理 vhost，后端仍为旧 64444 和新回环 16444。内部 HTTPS 分别按 fu101.gembox-sub.com、c2.deeplink-subscribe.com 的既有有效证书校验，不使用 proxy_ssl_verify off。 + +旧管理入口的私有路径在服务器运行时从既有容器配置提取，并与旧 vhost 比对，不写入 Git、CoDocs 或本文。源站只信任白色机传来的清洗后 X-Forwarded-For；直接请求的伪造标头不受信任。新增源域名证书使用独立 Certbot 目录和续期钩子，仅检查并重载宿主 Nginx，不运行原订阅网关的全局证书钩子。 + +白色机新橙云 vhost 单独引入官方 CF IP 信任文件，再向上游发送清洗后的真实 IP。新代理保留严格上游证书校验、路径、查询参数、WebSocket；Origin/Referer 只在与当前请求主机一致时转换，跨站值保留，避免放宽 CSRF 判断。Cookie、重定向和正文资源域名按实际入口复核；不盲目移除 CSP、CORS 或其他安全标头。 + +Cloudflare Origin CA 只适用于两个保持橙云的代理区域。用户浏览器使用 CF 边缘的公开可信证书；关闭橙云后直连白色机会不受浏览器信任。旧 com.de 不依赖此证书，须用公开可信证书。源站 CA 到期日须登记，不能假称它由 Certbot 自动续期。 + +## 隔离证书与运维入口 + +- gateway-01：`issue-gateway-admin.sh` 使用既有已接受协议的 LE 账户的同机副本，仅以 webroot 签发两个新增域名；`gateway-admin-cert.timer` 每日检查，隔离目录 `/etc/gateway-admin-letsencrypt`。不读取或运行原全局续期钩子。 +- 白色机：`install-origin-cert.py` 校验公开证书 SAN、有效期和服务器本地私钥公钥匹配，再安装两个 CF 源站证书；`white-proxy-cloudflare-ips.timer` 只更新新橙云站点引用的官方 IP 信任文件。 +- 旧 com.de：`issue-white-legacy.sh` 使用隔离目录 `/etc/white-proxy-letsencrypt`；`legacy-dns-hook.py` 只接受旧根域/一级泛域的 ACME 挑战，仅写委托目标 TXT，确认权威 DNS 传播后再继续，清理只删除自身创建且内容未改变的 ID。`white-proxy-cert.timer` 每日检查。 +- DNS 私有配置仅在白色机：`/etc/white-proxy-dns/config.json` 保存 zone_id/target，`cf-token` 保存令牌，均 root 所有、0600；Git 和普通文档不得保存令牌。CF 权限本身是整个 tomshell.online 区域 DNS 编辑，并非单 TXT 权限，另有白色机 IPv4 限制。用户于本轮明确批准创建与保存这一凭据。 +- 旧 com.de 证书续期依赖该 CNAME、CF 令牌和白色机出口 IP。变更这些资源先更新私有配置，再验证续期；不能删除其他 TXT、改变 NS 或创建无限制账户令牌。正式上线后必须执行隔离 `certbot renew --dry-run --no-directory-hooks` 验证自动续期。 + +## Git 发布与回退 + +发布源码先按项目正常 Git 流程提交/推送，只包含本次脚本和维护正文，保留工作区原有 AGENTS/DEPLOYMENT/README 修改。服务器检出明确运维提交；凭据、私钥、CSR 临时产物和备份不进入 Git。必要时使用已推送提交的 Git bundle fetch，不打包覆盖项目。 + +deploy.py 只管理列出的独立 Nginx 文件；原文件有精确备份与前后哈希，手工修改的管理文件或未被接管的同名文件会阻止覆盖。旧泛域配置只有在公开可信泛域证书到位并提供当前文件精确哈希时才撤回；保留原精确 master 匹配和其他专用配置。Nginx 检查/重载失败自动恢复本次文件并重新校验，不能操作原 Compose 的 down/update 或重启原容器。 + +回退时先核对当前哈希仍为本轮结果，按服务器私有备份仅恢复本轮文件。新证书账户、目录、定时器与旧实例隔离；撤销新域名入口不能撤销原订阅域名。回退或尚未完成的项目同任务同步 CoDocs、本文和 STATUS。 diff --git a/scripts/gateway-admin-proxy/.gitignore b/scripts/gateway-admin-proxy/.gitignore new file mode 100644 index 0000000..7a60b85 --- /dev/null +++ b/scripts/gateway-admin-proxy/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/scripts/gateway-admin-proxy/deploy.py b/scripts/gateway-admin-proxy/deploy.py new file mode 100644 index 0000000..973647c --- /dev/null +++ b/scripts/gateway-admin-proxy/deploy.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Install only approved proxy files, backing up and reverting on Nginx failure.""" +import argparse +import datetime +import hashlib +import importlib.util +import ipaddress +import json +import os +import pathlib +import re +import shutil +import subprocess + +spec = importlib.util.spec_from_file_location('render', pathlib.Path(__file__).with_name('render.py')) +render = importlib.util.module_from_spec(spec) +spec.loader.exec_module(render) + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def remove_legacy_wildcards(text): + spans = [] + for m in re.finditer(r'(?m)^server\s*\{', text): + end, depth = m.end(), 1 + while end < len(text) and depth: + depth += (text[end] == '{') - (text[end] == '}') + end += 1 + block = text[m.start():end] + if re.search(r'(?m)^\s*server_name\s+\*\.tomshell\.com\.de\s*;', block): + spans.append((m.start(), end)) + if len(spans) != 2: + raise RuntimeError('Expected exactly two old wildcard server blocks; refusing ambiguous edit') + for start, end in reversed(spans): + text = text[:start] + text[end:] + return text + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('role', choices=['gateway', 'white']) + parser.add_argument('--stage', choices=['http', 'https'], required=True) + parser.add_argument('--skip-legacy', action='store_true') + parser.add_argument('--legacy-certificate', default='/etc/white-proxy-letsencrypt/live/tomshell.com.de') + parser.add_argument('--expected-legacy-sha256') + parser.add_argument('--white-ip', default='104.168.30.40') + args = parser.parse_args() + ipaddress.ip_address(args.white_ip) + if not re.fullmatch(r'/[A-Za-z0-9/._-]+', args.legacy_certificate): + parser.error('Invalid certificate directory') + if os.geteuid() != 0: + parser.error('Remote deployment requires root') + os.umask(0o077) + subprocess.run(['nginx', '-t'], check=True, capture_output=True) + state_path = pathlib.Path('/var/lib/gateway-admin-proxy/state.json') + state_path.parent.mkdir(parents=True, mode=0o700, exist_ok=True) + old_state = json.loads(state_path.read_text()) if state_path.exists() else {'files': {}} + if old_state.get('role', args.role) != args.role: + parser.error('Deployment role differs from existing state') + if args.role == 'gateway': + legacy = '' + if args.stage == 'https': + data = json.loads(subprocess.check_output(['docker', 'inspect', 'subscribe-admin'], text=True))[0] + values = dict(x.split('=', 1) for x in data['Config']['Env'] if '=' in x) + legacy = '/' + values['ADMIN_SECRET_PATH'].strip('/') + '/' + if not re.fullmatch(r'/[A-Za-z0-9_-]+/', legacy): + parser.error('Invalid existing admin prefix') + if 'location ^~ ' + legacy not in pathlib.Path('/etc/nginx/conf.d/g1.gembox-sub.com.conf').read_text(): + parser.error('Legacy admin prefix disagrees with current vhost') + files = render.render_gateway(args.stage, legacy, args.white_ip) + else: + files = render.render_white(args.stage, args.legacy_certificate, args.skip_legacy) + changes = {pathlib.Path('/etc/nginx/conf.d') / name: data.encode() for name, data in files.items()} + if args.role == 'white' and args.stage == 'https' and not args.skip_legacy: + legacy_file = pathlib.Path('/etc/nginx/conf.d/master.tomshell.my.conf') + current = legacy_file.read_bytes() + if str(legacy_file) in old_state['files']: + if sha(current) != old_state['files'][str(legacy_file)]: + parser.error('Legacy file changed after previous deployment') + changes[legacy_file] = current + else: + if not args.expected_legacy_sha256 or sha(current) != args.expected_legacy_sha256: + parser.error('Legacy file needs an exact current hash before modifying wildcard blocks') + changes[legacy_file] = remove_legacy_wildcards(current.decode()).encode() + if args.stage == 'https': + for path, data in changes.items(): + if path.name == 'master.tomshell.my.conf': + continue + for cert in re.findall(rb'(?m)^\s*ssl_certificate\s+([^;]+);', data): + subprocess.run(['openssl', 'x509', '-in', cert.decode(), '-checkend', '86400', '-noout'], + check=True, capture_output=True) + if path.name == 'white-old-proxy.conf': + san = subprocess.check_output(['openssl', 'x509', '-in', cert.decode(), '-noout', + '-ext', 'subjectAltName'], text=True) + if 'DNS:*.tomshell.com.de' not in san or 'DNS:tomshell.com.de' not in san: + parser.error('Legacy proxy requires root and wildcard SANs, not a single-host certificate') + for path, data in changes.items(): + if path.exists() and str(path) not in old_state['files']: + if path.name != 'master.tomshell.my.conf': + raise RuntimeError('Unmanaged file already exists: ' + str(path)) + elif path.exists() and sha(path.read_bytes()) != old_state['files'].get(str(path)): + raise RuntimeError('Managed file changed outside this deployment: ' + str(path)) + timestamp = datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%d-%H%M%S-%f') + backup = pathlib.Path('/root/codex-backups/gateway-admin-proxy') / (timestamp + '-' + args.role) + backup.mkdir(parents=True, mode=0o700) + originals = {} + modes = {} + for path in changes: + originals[str(path)] = path.read_bytes() if path.exists() else None + modes[str(path)] = path.stat().st_mode & 0o777 if path.exists() else 0o644 + if path.exists(): + shutil.copy2(path, backup / path.name) + manifest = {'role': args.role, 'stage': args.stage, + 'before': {p: sha(data) if data is not None else None for p, data in originals.items()}, + 'after': {str(p): sha(data) for p, data in changes.items()}, 'modes': modes} + (backup / 'manifest.json').write_text(json.dumps(manifest, indent=2)) + try: + for path, data in changes.items(): + temp = path.with_suffix('.conf.new') + temp.write_bytes(data) + temp.chmod(modes[str(path)]) + os.replace(temp, path) + pathlib.Path(render.ACME).mkdir(parents=True, mode=0o755, exist_ok=True) + subprocess.run(['nginx', '-t'], check=True, capture_output=True) + subprocess.run(['systemctl', 'reload', 'nginx'], check=True) + except Exception: + for path, data in originals.items(): + target = pathlib.Path(path) + if data is None: + target.unlink(missing_ok=True) + else: + target.write_bytes(data) + target.chmod(modes[path]) + subprocess.run(['nginx', '-t'], check=True, capture_output=True) + subprocess.run(['systemctl', 'reload', 'nginx'], check=True) + raise + old_state.update({'role': args.role, 'stage': args.stage, 'backup': str(backup)}) + old_state['files'].update(manifest['after']) + state_new = state_path.with_suffix('.json.new') + state_new.write_text(json.dumps(old_state, indent=2)) + os.replace(state_new, state_path) + print(json.dumps({'role': args.role, 'stage': args.stage, 'backup': str(backup), + 'installed': [str(x) for x in changes], 'nginx_valid': True, 'reloaded': True})) + + +if __name__ == '__main__': + main() diff --git a/scripts/gateway-admin-proxy/gateway-admin-cert.service b/scripts/gateway-admin-proxy/gateway-admin-cert.service new file mode 100644 index 0000000..1637cca --- /dev/null +++ b/scripts/gateway-admin-proxy/gateway-admin-cert.service @@ -0,0 +1,5 @@ +[Unit] +Description=Renew only the two gateway administration host certificates +[Service] +Type=oneshot +ExecStart=/bin/sh /opt/gateway-admin-proxy/provision/renew-gateway-admin.sh diff --git a/scripts/gateway-admin-proxy/gateway-admin-cert.timer b/scripts/gateway-admin-proxy/gateway-admin-cert.timer new file mode 100644 index 0000000..65bc261 --- /dev/null +++ b/scripts/gateway-admin-proxy/gateway-admin-cert.timer @@ -0,0 +1,8 @@ +[Unit] +Description=Daily isolated gateway admin certificate renewal +[Timer] +OnCalendar=*-*-* 03:40:00 +RandomizedDelaySec=1800 +Persistent=true +[Install] +WantedBy=timers.target diff --git a/scripts/gateway-admin-proxy/install-origin-cert.py b/scripts/gateway-admin-proxy/install-origin-cert.py new file mode 100644 index 0000000..514bddd --- /dev/null +++ b/scripts/gateway-admin-proxy/install-origin-cert.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Install the public CF certificate only after matching the server-held key and SANs.""" +import argparse +import os +import pathlib +import subprocess + +p = argparse.ArgumentParser() +p.add_argument('domain', choices=['tomshell.online', 'tomtojapan.online']) +p.add_argument('certificate', type=pathlib.Path) +a = p.parse_args() +os.umask(0o077) +root = pathlib.Path('/etc/white-proxy-origin') / a.domain +key = root / 'privatekey.pem' +if not key.exists(): + p.error('Generate the key and CSR on the white machine first') +subprocess.run(['openssl', 'x509', '-in', str(a.certificate), '-checkend', '86400', '-noout'], + check=True, capture_output=True) +san = subprocess.check_output(['openssl', 'x509', '-in', str(a.certificate), '-noout', '-ext', + 'subjectAltName'], text=True) +if 'DNS:*.' + a.domain not in san or 'DNS:' + a.domain not in san: + p.error('Certificate does not contain the expected root and wildcard names') +cert_pub = subprocess.check_output(['openssl', 'x509', '-in', str(a.certificate), '-noout', '-pubkey']) +key_pub = subprocess.check_output(['openssl', 'pkey', '-in', str(key), '-pubout']) +if cert_pub.strip() != key_pub.strip(): + p.error('Certificate public key differs from the server-held private key') +target = root / 'fullchain.pem' +if target.exists() and target.read_bytes() != a.certificate.read_bytes(): + p.error('Different certificate already exists; take an explicit backup before replacement') +temporary = root / 'fullchain.pem.new' +temporary.write_bytes(a.certificate.read_bytes()) +temporary.chmod(0o600) +os.replace(temporary, target) +print('Installed matching public origin certificate: ' + a.domain + '; private key retained on server') diff --git a/scripts/gateway-admin-proxy/issue-gateway-admin.sh b/scripts/gateway-admin-proxy/issue-gateway-admin.sh new file mode 100644 index 0000000..9b9b9f9 --- /dev/null +++ b/scripts/gateway-admin-proxy/issue-gateway-admin.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# Reuse the already accepted gateway LE account without running global hooks. +set -eu +umask 077 +test "$(id -u)" = 0 +test -d /etc/dpl-letsencrypt/accounts/acme-v02.api.letsencrypt.org +install -d -m 700 /etc/gateway-admin-letsencrypt /var/lib/gateway-admin-proxy/certbot /var/log/gateway-admin-certbot +if [ ! -d /etc/gateway-admin-letsencrypt/accounts ]; then + cp -a /etc/dpl-letsencrypt/accounts /etc/gateway-admin-letsencrypt/accounts +fi +exec certbot certonly --non-interactive --webroot \ + --webroot-path /var/lib/gateway-admin-proxy/acme \ + --config-dir /etc/gateway-admin-letsencrypt \ + --work-dir /var/lib/gateway-admin-proxy/certbot \ + --logs-dir /var/log/gateway-admin-certbot \ + --no-directory-hooks --cert-name gateway-admin \ + -d getway-fuc.gemcook.net -d getway-dpl.gemcook.net diff --git a/scripts/gateway-admin-proxy/issue-white-legacy.sh b/scripts/gateway-admin-proxy/issue-white-legacy.sh new file mode 100644 index 0000000..ec51e12 --- /dev/null +++ b/scripts/gateway-admin-proxy/issue-white-legacy.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -eu +umask 077 +test "$(id -u)" = 0 +test -d /etc/letsencrypt/accounts/acme-v02.api.letsencrypt.org +install -d -m 700 /etc/white-proxy-letsencrypt /var/lib/white-proxy-certbot /var/log/white-proxy-certbot +if [ ! -d /etc/white-proxy-letsencrypt/accounts ]; then + cp -a /etc/letsencrypt/accounts /etc/white-proxy-letsencrypt/accounts +fi +exec certbot certonly --non-interactive --manual --preferred-challenges dns \ + --manual-auth-hook 'python3 /opt/gateway-admin-proxy/provision/legacy-dns-hook.py auth' \ + --manual-cleanup-hook 'python3 /opt/gateway-admin-proxy/provision/legacy-dns-hook.py cleanup' \ + --config-dir /etc/white-proxy-letsencrypt \ + --work-dir /var/lib/white-proxy-certbot \ + --logs-dir /var/log/white-proxy-certbot \ + --no-directory-hooks --cert-name tomshell.com.de \ + -d tomshell.com.de -d '*.tomshell.com.de' diff --git a/scripts/gateway-admin-proxy/legacy-dns-hook.py b/scripts/gateway-admin-proxy/legacy-dns-hook.py new file mode 100644 index 0000000..d57c475 --- /dev/null +++ b/scripts/gateway-admin-proxy/legacy-dns-hook.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Certbot DNS hook: only one approved delegation, exact TXT IDs, IPv4 API.""" +import hashlib +import json +import os +import pathlib +import re +import subprocess +import sys +import tempfile +import time +import urllib.parse + +CONFIG = pathlib.Path('/etc/white-proxy-dns/config.json') +TOKEN = pathlib.Path('/etc/white-proxy-dns/cf-token') +STATE = pathlib.Path('/var/lib/white-proxy-dns') +DOMAIN = 'tomshell.com.de' +TARGET = 'old-com-de.acme.tomshell.online' +AUTH_NS = ('jaziel.ns.cloudflare.com', 'nena.ns.cloudflare.com') + + +def check_private(path): + if path.stat().st_uid != 0 or path.stat().st_mode & 0o077: + raise RuntimeError('DNS credentials must be root-owned and private') + + +def api(method, suffix, payload=None): + check_private(TOKEN) + check_private(CONFIG) + cfg = json.loads(CONFIG.read_text()) + zone = cfg['zone_id'] + if not re.fullmatch(r'[a-f0-9]{32}', zone) or cfg['target'] != TARGET: + raise RuntimeError('Unapproved DNS configuration') + token = TOKEN.read_text().strip() + if not re.fullmatch(r'[A-Za-z0-9_-]{40}', token): + raise RuntimeError('Invalid token format') + with tempfile.TemporaryDirectory(prefix='acme-api-', dir=STATE) as folder: + body = pathlib.Path(folder) / 'request.json' + args = ['curl', '-4', '--silent', '--show-error', '--fail-with-body', + '--connect-timeout', '10', '--max-time', '30', '--config', '-', + '--request', method, + 'https://api.cloudflare.com/client/v4/zones/' + zone + '/dns_records' + suffix] + if payload is not None: + body.write_text(json.dumps(payload)) + args.extend(['--data-binary', '@' + str(body)]) + config = 'header = "Authorization: Bearer ' + token + '"\nheader = "Content-Type: application/json"\n' + result = subprocess.run(args, input=config, capture_output=True, text=True) + if result.returncode: + raise RuntimeError('Cloudflare DNS API request failed; response intentionally not logged') + value = json.loads(result.stdout) + if not value.get('success'): + raise RuntimeError('Cloudflare DNS API rejected request') + return value['result'] + + +def dns(name, kind, server): + return subprocess.check_output(['dig', '-4', '+time=3', '+tries=1', '+short', + '@' + server, name, kind], text=True).strip() + + +def matches(record, validation): + return record.get('type') == 'TXT' and record.get('name') == TARGET and record.get('content') == validation + + +def main(): + if os.geteuid() != 0 or len(sys.argv) != 2 or sys.argv[1] not in ('auth', 'cleanup'): + raise RuntimeError('Root auth/cleanup mode required') + os.umask(0o077) + STATE.mkdir(parents=True, mode=0o700, exist_ok=True) + domain = os.environ['CERTBOT_DOMAIN'].removeprefix('*.') + validation = os.environ['CERTBOT_VALIDATION'] + if domain != DOMAIN or not re.fullmatch(r'[A-Za-z0-9_-]{43}', validation): + raise RuntimeError('Challenge outside approved domain') + state = STATE / (hashlib.sha256(validation.encode()).hexdigest() + '.json') + if sys.argv[1] == 'cleanup': + if not state.exists(): + return + record_id = json.loads(state.read_text())['id'] + if not re.fullmatch(r'[a-f0-9]{32}', record_id): + raise RuntimeError('Invalid TXT record ID') + record = api('GET', '/' + record_id) + if not matches(record, validation): + raise RuntimeError('TXT record changed; refusing deletion') + api('DELETE', '/' + record_id) + state.unlink() + return + for ns in ('ns1.dns-parking.com', 'ns2.dns-parking.com'): + if dns('_acme-challenge.' + DOMAIN, 'CNAME', ns).rstrip('.') != TARGET: + raise RuntimeError('Approved CNAME delegation not published') + if state.exists(): + raise RuntimeError('Existing challenge state; inspect before retry') + record = api('POST', '', {'type': 'TXT', 'name': TARGET, 'content': validation, 'ttl': 60, + 'comment': 'Isolated legacy white-proxy ACME challenge'}) + state.write_text(json.dumps({'id': record['id']})) + try: + deadline = time.monotonic() + 600 + while time.monotonic() < deadline: + if all('"' + validation + '"' in dns(TARGET, 'TXT', ns).splitlines() for ns in AUTH_NS): + # Also allow recursors to replace a cached negative answer before validation. + for _ in range(24): + if all('"' + validation + '"' in dns('_acme-challenge.' + DOMAIN, 'TXT', ns).splitlines() + for ns in ('1.1.1.1', '8.8.8.8')): + return + time.sleep(5) + raise RuntimeError('Challenge not visible through public DNS recursors') + time.sleep(5) + raise RuntimeError('Challenge propagation timed out') + except Exception: + if matches(api('GET', '/' + record['id']), validation): + api('DELETE', '/' + record['id']) + state.unlink() + raise + + +if __name__ == '__main__': + main() diff --git a/scripts/gateway-admin-proxy/render.py b/scripts/gateway-admin-proxy/render.py new file mode 100644 index 0000000..cad5cbb --- /dev/null +++ b/scripts/gateway-admin-proxy/render.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +"""Render the user-approved admin hostnames and bounded white-machine proxies.""" +import argparse +import ipaddress +import os +import pathlib +import re + +CA = '/etc/ssl/certs/ca-certificates.crt' +ACME = '/var/lib/gateway-admin-proxy/acme' +CF_TRUST = '/etc/nginx/white-proxy-cloudflare-realip.conf' + + +def proxy_lines(upstream, tls_name, host, alias=None): + result = f''' proxy_pass https://{upstream}$request_uri; + proxy_ssl_server_name on; + proxy_ssl_name {tls_name}; + proxy_ssl_verify on; + proxy_ssl_verify_depth 4; + proxy_ssl_trusted_certificate {CA}; + proxy_http_version 1.1; + proxy_set_header Host {host}; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $gateway_admin_proxy_connection; + proxy_connect_timeout 15s; + proxy_read_timeout 120s; + proxy_cache_bypass 1; + proxy_no_cache 1; +''' + if alias: + result += f''' proxy_set_header X-Forwarded-Host $host; + proxy_set_header Origin $gateway_admin_origin; + proxy_set_header Referer $gateway_admin_referer; + proxy_redirect https://{host}/ https://$host/; + proxy_cookie_domain {host} $host; +''' + else: + result += ' proxy_set_header X-Forwarded-Host $host;\n' + return result + + +def http_server(names, redirect=True): + fallback = 'return 301 https://$host$request_uri;' if redirect else 'return 404;' + return f'''server {{ + listen 80; + listen [::]:80; + server_name {names}; + access_log off; + location ^~ /.well-known/acme-challenge/ {{ + root {ACME}; + try_files $uri =404; + }} + location / {{ {fallback} }} +}} +''' + + +def https_server(names, certificate, content, cloudflare=False, white_ip=None, key_name='privatekey.pem'): + trusted = f' include {CF_TRUST};\n' if cloudflare else '' + if white_ip: + trusted += f''' set_real_ip_from {white_ip}; + real_ip_header X-Forwarded-For; + real_ip_recursive on; +''' + return f'''server {{ + listen 443 ssl; + listen [::]:443 ssl; + server_name {names}; + ssl_certificate {certificate}/fullchain.pem; + ssl_certificate_key {certificate}/{key_name}; + ssl_protocols TLSv1.2 TLSv1.3; + client_max_body_size 20m; + access_log off; + error_log /var/log/nginx/gateway-admin-proxy.error.log warn; +{trusted}{content} +}} +''' + + +def render_gateway(stage, legacy_path, white_ip): + files = {'00-gateway-admin-proxy-maps.conf': '''map $http_upgrade $gateway_admin_proxy_connection { + default upgrade; + '' close; +} +'''} + for kind, port, tls in [('fuc', 64444, 'fu101.gembox-sub.com'), + ('dpl', 16444, 'c2.deeplink-subscribe.com')]: + name = f'getway-{kind}.gemcook.net' + content = http_server(name, stage == 'https') + if stage == 'https': + if kind == 'fuc': + location = f''' location = / {{ return 302 {legacy_path}; }} + location ^~ {legacy_path} {{ +{proxy_lines('127.0.0.1:'+str(port), tls, name)} }} + location / {{ return 404; }} +''' + else: + location = f''' location / {{ +{proxy_lines('127.0.0.1:'+str(port), tls, name)} }} +''' + # The admin upstreams retain their own certificates and authentication. + location += ' add_header Cache-Control "private, no-store" always;\n' + content += https_server(name, '/etc/gateway-admin-letsencrypt/live/gateway-admin', + location, white_ip=white_ip, key_name='privkey.pem') + files[f'gateway-admin-{kind}.conf'] = content + return files + + +def render_white(stage, legacy_certificate, skip_legacy=False): + files = {} + maps = '''map $http_upgrade $gateway_admin_proxy_connection { + default upgrade; + '' close; +} +map $host $gateway_admin_origin_host { + tomshell.online gemcook.net; + ~^(?[a-z0-9-]+)\\.tomshell\\.online$ $gnew.gemcook.net; + tomshell.com.de diaodu.us; + ~^(?[a-z0-9-]+)\\.tomshell\\.com\\.de$ $gold.diaodu.us; + dpl.tomtojapan.online dp-link.net; + fuc.tomtojapan.online xn--9kqy16h.com; + default ''; +} +map "$host|$http_origin" $gateway_admin_origin { + "~^(?[a-z0-9.-]+)\\|https://\\k$" https://$gateway_admin_origin_host; + default $http_origin; +} +map "$host|$http_referer" $gateway_admin_referer { + "~^(?[a-z0-9.-]+)\\|https://\\k(?/.*)$" https://$gateway_admin_origin_host$grefpath; + default $http_referer; +} +''' + files['00-gateway-admin-proxy-maps.conf'] = maps + specs = [ + ('new', 'tomshell.online *.tomshell.online', '/etc/white-proxy-origin/tomshell.online', + 'gemcook.net', 'tomshell.online', True), + ('old', 'tomshell.com.de *.tomshell.com.de', legacy_certificate, + 'diaodu.us', 'tomshell.com.de', False), + ('dpl-site', 'dpl.tomtojapan.online', '/etc/white-proxy-origin/tomtojapan.online', + 'dp-link.net', 'dpl.tomtojapan.online', True), + ('fuc-site', 'fuc.tomtojapan.online', '/etc/white-proxy-origin/tomtojapan.online', + 'xn--9kqy16h.com', 'fuc.tomtojapan.online', True), + ] + for key, names, cert, source, alias, cf in specs: + if key == 'old' and skip_legacy: + continue + data = http_server(names, stage == 'https') + if stage == 'https': + extras = '' + if key in ('new', 'old'): + source_regex = re.escape(source) + extras = f''' proxy_redirect ~^https?://([a-z0-9-]+\\.)?{source_regex}(/.*)$ https://$1{alias}$2; + proxy_cookie_domain {source} {alias}; +''' + elif key == 'fuc-site': + extras = " sub_filter '福云.com' 'fuc.tomtojapan.online';\n" + location = f''' resolver 1.1.1.1 1.0.0.1 valid=300s ipv6=off; + resolver_timeout 5s; + if ($gateway_admin_origin_host = '') {{ return 444; }} + location / {{ +{proxy_lines('$gateway_admin_origin_host', '$gateway_admin_origin_host', '$gateway_admin_origin_host', alias)}{extras} + proxy_set_header Accept-Encoding ""; + sub_filter_once off; + sub_filter_types text/css application/javascript application/json; + sub_filter '{source}' '{alias}'; + }} +''' + data += https_server(names, cert, location, cloudflare=cf, + key_name='privatekey.pem' if cf else 'privkey.pem') + files[f'white-{key}-proxy.conf'] = data + return files + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('role', choices=['gateway', 'white']) + p.add_argument('--stage', choices=['http', 'https'], required=True) + p.add_argument('--output', type=pathlib.Path, required=True) + p.add_argument('--white-ip', default='104.168.30.40') + p.add_argument('--legacy-certificate', default='/etc/white-proxy-letsencrypt/live/tomshell.com.de') + p.add_argument('--skip-legacy', action='store_true', help='Leave legacy proxy untouched until its wildcard certificate is ready') + a = p.parse_args() + ipaddress.ip_address(a.white_ip) + if not re.fullmatch(r'/[A-Za-z0-9/._-]+', a.legacy_certificate): + p.error('Invalid certificate directory') + legacy_path = os.environ.get('LEGACY_ADMIN_PATH', '') + if a.role == 'gateway' and a.stage == 'https' and not re.fullmatch(r'/[A-Za-z0-9_-]+/', legacy_path): + p.error('Supply the existing legacy admin prefix through LEGACY_ADMIN_PATH') + files = render_gateway(a.stage, legacy_path, a.white_ip) if a.role == 'gateway' else render_white(a.stage, a.legacy_certificate, a.skip_legacy) + a.output.mkdir(parents=True, exist_ok=True) + for name, content in files.items(): + (a.output / name).write_text(content) + print(f'Rendered {len(files)} {a.role}/{a.stage} files; no service configuration changed') + + +if __name__ == '__main__': + main() diff --git a/scripts/gateway-admin-proxy/renew-gateway-admin.sh b/scripts/gateway-admin-proxy/renew-gateway-admin.sh new file mode 100644 index 0000000..d5b901e --- /dev/null +++ b/scripts/gateway-admin-proxy/renew-gateway-admin.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +umask 077 +exec certbot renew --quiet \ + --config-dir /etc/gateway-admin-letsencrypt \ + --work-dir /var/lib/gateway-admin-proxy/certbot \ + --logs-dir /var/log/gateway-admin-certbot \ + --no-directory-hooks \ + --deploy-hook '/usr/sbin/nginx -t && /bin/systemctl reload nginx' diff --git a/scripts/gateway-admin-proxy/renew-white-legacy.sh b/scripts/gateway-admin-proxy/renew-white-legacy.sh new file mode 100644 index 0000000..517bc95 --- /dev/null +++ b/scripts/gateway-admin-proxy/renew-white-legacy.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +umask 077 +exec certbot renew --quiet \ + --config-dir /etc/white-proxy-letsencrypt \ + --work-dir /var/lib/white-proxy-certbot \ + --logs-dir /var/log/white-proxy-certbot \ + --no-directory-hooks \ + --deploy-hook '/usr/sbin/nginx -t && /bin/systemctl reload nginx' diff --git a/scripts/gateway-admin-proxy/sync-cf-ips.py b/scripts/gateway-admin-proxy/sync-cf-ips.py new file mode 100644 index 0000000..99746d5 --- /dev/null +++ b/scripts/gateway-admin-proxy/sync-cf-ips.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Synchronize only the new white proxy vhosts' Cloudflare trust include.""" +import ipaddress +import os +import pathlib +import subprocess + +target = pathlib.Path('/etc/nginx/white-proxy-cloudflare-realip.conf') +ranges = [] +for version in (4, 6): + p = subprocess.run(['curl', '-4', '--fail', '--silent', '--show-error', '--max-time', '30', + f'https://www.cloudflare.com/ips-v{version}'], + check=True, capture_output=True, text=True) + nets = [ipaddress.ip_network(x.strip(), strict=True) for x in p.stdout.splitlines() if x.strip()] + if not ((5 if version == 4 else 2) <= len(nets) <= 100): + raise RuntimeError('Unexpected Cloudflare range count') + if any(n.version != version or n.prefixlen < (8 if version == 4 else 16) for n in nets): + raise RuntimeError('Invalid or overbroad Cloudflare trust range') + ranges.extend(str(n) for n in nets) +if len(ranges) != len(set(ranges)): + raise RuntimeError('Duplicate Cloudflare ranges') +data = ('# Managed new white proxy vhosts only; Cloudflare official ranges.\n' + + ''.join(f'set_real_ip_from {n};\n' for n in ranges) + + 'real_ip_header CF-Connecting-IP;\nreal_ip_recursive off;\n').encode() +old = target.read_bytes() if target.exists() else None +if old == data: + print('White proxy Cloudflare trust unchanged') + raise SystemExit(0) +temporary = target.with_suffix('.conf.new') +temporary.write_bytes(data) +temporary.chmod(0o644) +os.replace(temporary, target) +try: + subprocess.run(['nginx', '-t'], check=True, capture_output=True) + subprocess.run(['systemctl', 'reload', 'nginx'], check=True) +except Exception: + if old is None: + target.unlink() + else: + temporary.write_bytes(old) + os.replace(temporary, target) + subprocess.run(['nginx', '-t'], check=True, capture_output=True) + subprocess.run(['systemctl', 'reload', 'nginx'], check=True) + raise +print(f'White proxy Cloudflare trust synchronized: {len(ranges)} ranges') diff --git a/scripts/gateway-admin-proxy/verify-maps.py b/scripts/gateway-admin-proxy/verify-maps.py new file mode 100644 index 0000000..0e102cd --- /dev/null +++ b/scripts/gateway-admin-proxy/verify-maps.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Exercise real Nginx mapping and Origin/Referer isolation using localhost only.""" +import importlib.util +import pathlib +import shutil +import socket +import subprocess +import tempfile +import time +import urllib.request + +spec = importlib.util.spec_from_file_location('render', pathlib.Path(__file__).with_name('render.py')) +render = importlib.util.module_from_spec(spec) +spec.loader.exec_module(render) +nginx = shutil.which('nginx') +if not nginx: + raise SystemExit('Nginx is required for this isolated integration check') + +with tempfile.TemporaryDirectory(prefix='gateway-proxy-maps-') as folder: + root = pathlib.Path(folder) + with socket.socket() as sock: + sock.bind(('127.0.0.1', 0)) + port = sock.getsockname()[1] + maps = render.render_white('https', '/unused')['00-gateway-admin-proxy-maps.conf'] + config = root / 'nginx.conf' + config.write_text(f'''daemon off; +pid {root}/nginx.pid; +error_log {root}/error.log; +events {{ worker_connections 64; }} +http {{ + access_log off; +{maps} + server {{ + listen 127.0.0.1:{port}; + location / {{ + return 200 "$gateway_admin_origin_host|$gateway_admin_origin|$gateway_admin_referer"; + }} + }} +}} +''') + subprocess.run([nginx, '-t', '-c', str(config), '-p', folder], check=True, + capture_output=True, text=True) + process = subprocess.Popen([nginx, '-c', str(config), '-p', folder], + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True) + try: + for _ in range(40): + try: + with socket.create_connection(('127.0.0.1', port), timeout=.1): + break + except OSError: + if process.poll() is not None: + raise RuntimeError(process.stderr.read()) + time.sleep(.05) + cases = [ + ('tomshell.online', 'gemcook.net'), + ('getway-fuc.tomshell.online', 'getway-fuc.gemcook.net'), + ('getway-dpl.tomshell.online', 'getway-dpl.gemcook.net'), + ('nasa.tomshell.com.de', 'nasa.diaodu.us'), + ('tomshell.com.de', 'diaodu.us'), + ('dpl.tomtojapan.online', 'dp-link.net'), + ('fuc.tomtojapan.online', 'xn--9kqy16h.com'), + ('not-registered.example', ''), + ('two.levels.tomshell.online', ''), + ] + assertions = 0 + for host, upstream in cases: + for origin in ['https://' + host, 'https://other.tomshell.online', 'https://outside.example']: + referer = origin + '/example?keep=1' + req = urllib.request.Request('http://127.0.0.1:' + str(port) + '/', + headers={'Host': host, 'Origin': origin, 'Referer': referer}) + with urllib.request.urlopen(req, timeout=3) as response: + actual = response.read().decode().split('|') + assert actual[0] == upstream, (host, actual[0], upstream) + own = origin == 'https://' + host + assert actual[1] == ('https://' + upstream if own else origin), (host, origin, actual[1]) + assert actual[2] == ('https://' + upstream + '/example?keep=1' if own else referer) + assertions += 3 + print(f'PASS: {assertions} real-Nginx map/header checks; localhost fixture only, no origin requests') + finally: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() diff --git a/scripts/gateway-admin-proxy/white-proxy-cert.service b/scripts/gateway-admin-proxy/white-proxy-cert.service new file mode 100644 index 0000000..773c8bc --- /dev/null +++ b/scripts/gateway-admin-proxy/white-proxy-cert.service @@ -0,0 +1,5 @@ +[Unit] +Description=Renew isolated public wildcard certificate for legacy white proxy +[Service] +Type=oneshot +ExecStart=/bin/sh /opt/gateway-admin-proxy/provision/renew-white-legacy.sh diff --git a/scripts/gateway-admin-proxy/white-proxy-cert.timer b/scripts/gateway-admin-proxy/white-proxy-cert.timer new file mode 100644 index 0000000..cea5769 --- /dev/null +++ b/scripts/gateway-admin-proxy/white-proxy-cert.timer @@ -0,0 +1,8 @@ +[Unit] +Description=Daily renewal check for legacy white proxy certificate +[Timer] +OnCalendar=*-*-* 04:10:00 +RandomizedDelaySec=1800 +Persistent=true +[Install] +WantedBy=timers.target diff --git a/scripts/gateway-admin-proxy/white-proxy-cloudflare-ips.service b/scripts/gateway-admin-proxy/white-proxy-cloudflare-ips.service new file mode 100644 index 0000000..037917c --- /dev/null +++ b/scripts/gateway-admin-proxy/white-proxy-cloudflare-ips.service @@ -0,0 +1,5 @@ +[Unit] +Description=Synchronize CF ranges for the new white proxy vhosts +[Service] +Type=oneshot +ExecStart=/usr/bin/python3 /opt/gateway-admin-proxy/provision/sync-cf-ips.py diff --git a/scripts/gateway-admin-proxy/white-proxy-cloudflare-ips.timer b/scripts/gateway-admin-proxy/white-proxy-cloudflare-ips.timer new file mode 100644 index 0000000..a41121b --- /dev/null +++ b/scripts/gateway-admin-proxy/white-proxy-cloudflare-ips.timer @@ -0,0 +1,8 @@ +[Unit] +Description=Daily white proxy CF trust refresh +[Timer] +OnCalendar=*-*-* 02:30:00 +RandomizedDelaySec=900 +Persistent=true +[Install] +WantedBy=timers.target