ops: isolate gateway certificate renewal and preserve client IP

dpl-gateway-cert.service

@@ -0,0 +1,5 @@
+[Unit]
+Description=Renew isolated DPL subscription gateway certificate
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/certbot renew --config-dir /etc/dpl-letsencrypt --work-dir /var/lib/dpl-sgw/certbot --logs-dir /var/log/dpl-certbot --no-directory-hooks --quiet --deploy-hook /opt/dpl-gateway/provision/renew-gateway.sh

dpl-gateway-cert.timer

@@ -0,0 +1,8 @@
+[Unit]
+Description=Daily isolated DPL gateway certificate renewal
+[Timer]
+OnCalendar=*-*-* 03:00:00
+RandomizedDelaySec=3600
+Persistent=true
+[Install]
+WantedBy=timers.target

gateway-admin.conf

@@ -8,8 +8,8 @@ http {
     server_tokens off;
     server {
         listen 64444 ssl;
-        ssl_certificate /etc/letsencrypt/live/c2.deeplink-subscribe.com/fullchain.pem;
-        ssl_certificate_key /etc/letsencrypt/live/c2.deeplink-subscribe.com/privkey.pem;
+        ssl_certificate /etc/dpl-letsencrypt/live/c2.deeplink-subscribe.com/fullchain.pem;
+        ssl_certificate_key /etc/dpl-letsencrypt/live/c2.deeplink-subscribe.com/privkey.pem;
         root /var/www/html;
         index index.php;
         location / { try_files $uri /index.php$is_args$args; }

gateway-nginx.conf

@@ -8,7 +8,7 @@ http {
     default_type application/octet-stream;
     include /etc/nginx/subscribe/real_ip.conf;
     # Do not record subscription credentials in access logs.
-    log_format main '$remote_addr [$time_local] "$request_method subscription" $status $body_bytes_sent "$http_user_agent"';
+    log_format main '$remote_addr [$time_local] "$request_method $uri $server_protocol" $status $body_bytes_sent "$http_user_agent"';
     access_log /var/log/subscribe/access.log main;
     error_log /var/log/subscribe/error.log crit;
     sendfile on;

gateway-protect.conf

@@ -0,0 +1,26 @@
+location ^~ /api/v1/client/subscribe {
+    if ($is_cloud_ip = 1) { set $block_reason "cloud"; }
+    if ($bad_subscribe_ua = 1) { set $block_reason "ua"; }
+    if ($is_custom_bad_ua = 1) { set $block_reason "ua"; }
+    if ($is_ua_whitelisted = 1) { set $block_reason ""; }
+    if ($whitelist_ip = 1) { set $block_reason ""; }
+    if ($block_reason = "cloud") { return 403 "Forbidden: Cloud IP"; }
+    if ($block_reason = "ua") { return 403 "Forbidden: Invalid Client"; }
+    limit_req zone=subscribe_limit burst=5 nodelay;
+    limit_req_status 429;
+    set $upstream_backend ${V2B_BACKEND};
+    proxy_pass $upstream_backend;
+    proxy_set_header Host ${V2B_HOST};
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $remote_addr;
+    proxy_set_header X-Forwarded-Proto https;
+    proxy_ssl_server_name on;
+    proxy_ssl_name ${V2B_HOST};
+    proxy_ssl_verify on;
+    proxy_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt;
+    proxy_http_version 1.1;
+    proxy_connect_timeout 10s;
+    proxy_read_timeout 60s;
+    add_header Cache-Control "private, no-store" always;
+    add_header X-Subscribe-Filter "active" always;
+}

gateway-site.conf

@@ -7,8 +7,8 @@ server {
 server {
     listen 443 ssl;
     server_name c2.deeplink-subscribe.com;
-    ssl_certificate /etc/letsencrypt/live/c2.deeplink-subscribe.com/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/c2.deeplink-subscribe.com/privkey.pem;
+    ssl_certificate /etc/dpl-letsencrypt/live/c2.deeplink-subscribe.com/fullchain.pem;
+    ssl_certificate_key /etc/dpl-letsencrypt/live/c2.deeplink-subscribe.com/privkey.pem;
     ssl_protocols TLSv1.2 TLSv1.3;
     # Public gateway serves subscriptions only; administration stays on loopback.
     access_log off;

gateway.compose.yml

@@ -18,9 +18,10 @@ services:
     volumes:
       - config:/etc/nginx/subscribe
       - logs:/var/log/subscribe
-      - /etc/letsencrypt:/etc/letsencrypt:ro
+      - /etc/dpl-letsencrypt:/etc/dpl-letsencrypt:ro
       - /var/lib/dpl-sgw/ssl:/etc/nginx/ssl:ro
       - ./gateway-nginx.conf:/etc/nginx/templates-src/nginx.conf:ro
+      - ./gateway-protect.conf:/etc/nginx/templates-src/subscribe_protect.conf.template:ro
     networks: [dpl]
     logging: {driver: json-file, options: {max-size: "5m", max-file: "2"}}
   admin:
@@ -36,7 +37,7 @@ services:
     volumes:
       - config:/etc/nginx/subscribe
       - logs:/var/log/subscribe
-      - /etc/letsencrypt:/etc/letsencrypt:ro
+      - /etc/dpl-letsencrypt:/etc/dpl-letsencrypt:ro
       - ./gateway-admin.conf:/etc/nginx/nginx.conf:ro
     networks: [dpl]
     logging: {driver: json-file, options: {max-size: "5m", max-file: "2"}}

install-gateway.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -euo pipefail
+[[ $(id -u) == 0 ]] || exit 1
+cd "$(dirname "$0")"
+test -s /etc/dpl-letsencrypt/live/c2.deeplink-subscribe.com/fullchain.pem
+docker image inspect sha256:d8807a5b2442bc25ce4fbc1312354b9dcbb0f538d75181ef735f4ea980760e70 >/dev/null
+docker image inspect sha256:31ff8ea3e8f958a835edbd1579709017da75b742ff3a5984ea59d12ea0775577 >/dev/null
+install -d -m 700 /var/lib/dpl-sgw
+install -d -m 755 /var/lib/dpl-sgw/acme /var/lib/dpl-sgw/ssl
+ln -sfn /etc/dpl-letsencrypt/live/c2.deeplink-subscribe.com/fullchain.pem /var/lib/dpl-sgw/ssl/cert.pem
+ln -sfn /etc/dpl-letsencrypt/live/c2.deeplink-subscribe.com/privkey.pem /var/lib/dpl-sgw/ssl/key.pem
+if [[ ! -e /var/lib/dpl-sgw/admin.env ]]; then
+    umask 077
+    printf 'ADMIN_USER=dpl-admin\nADMIN_PASS=%s\nV2B_BACKEND=https://dp-link.net\nV2B_HOST=dp-link.net\n' "$(openssl rand -hex 24)" > /var/lib/dpl-sgw/admin.env
+fi
+docker volume create dpl_sgw_config >/dev/null
+config_dir=$(docker volume inspect -f '{{.Mountpoint}}' dpl_sgw_config)
+# Only the new volume. Allow the documented jump host for acceptance checks.
+if [[ ! -f "$config_dir/whitelist_ips.txt" ]]; then
+    printf '104.168.30.40\n' > "$config_dir/whitelist_ips.txt"
+fi
+chmod 755 renew-gateway.sh
+docker compose -f gateway.compose.yml config --quiet
+docker compose -f gateway.compose.yml up -d
+install -m 644 dpl-gateway-cert.service dpl-gateway-cert.timer /etc/systemd/system/
+systemctl daemon-reload
+systemctl enable --now dpl-gateway-cert.timer
+echo 'New gateway started; verify initialization before enabling the HTTPS host site.'

isolate-gateway-cert.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+"""Move only the newly issued DPL certificate out of the old global renewal hooks."""
+import pathlib
+import shutil
+
+domain = "c2.deeplink-subscribe.com"
+old = pathlib.Path("/etc/letsencrypt")
+new = pathlib.Path("/etc/dpl-letsencrypt")
+renewal = old / "renewal" / (domain + ".conf")
+if not renewal.exists():
+    assert (new / "renewal" / (domain + ".conf")).exists(), "No DPL renewal registration"
+    print("DPL certificate already isolated")
+    raise SystemExit
+text = renewal.read_text()
+assert "authenticator = webroot" in text
+assert "https://acme-v02.api.letsencrypt.org/directory" in text
+account = next(line.split("=", 1)[1].strip() for line in text.splitlines() if line.startswith("account ="))
+assert account.isalnum()
+new.mkdir(mode=0o700, exist_ok=True)
+for folder in ("live", "archive", "renewal", "accounts"):
+    (new / folder).mkdir(mode=0o700, exist_ok=True)
+for folder in ("live", "archive"):
+    assert (old / folder / domain).is_dir()
+    assert not (new / folder / domain).exists(), "Partial migration; inspect before retry"
+account_relative = pathlib.Path("accounts/acme-v02.api.letsencrypt.org/directory") / account
+shutil.copytree(old / account_relative, new / account_relative, dirs_exist_ok=True)
+backup = pathlib.Path("/root/codex-backups/dpl-gateway-20261011/certificate-before-isolation")
+backup.mkdir(mode=0o700, exist_ok=True)
+shutil.copy2(renewal, backup / renewal.name)
+for folder in ("live", "archive"):
+    shutil.move(str(old / folder / domain), str(new / folder / domain))
+(new / "renewal" / renewal.name).write_text(text.replace("/etc/letsencrypt/", "/etc/dpl-letsencrypt/"))
+(new / "renewal" / renewal.name).chmod(0o600)
+renewal.unlink()  # Only the new DPL registration; old certificates and hooks stay intact.
+assert (new / "live" / domain / "fullchain.pem").is_file()
+print("DPL certificate isolated from original gateway renewal hooks")

renew-gateway.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -eu
+[ "${RENEWED_LINEAGE:-}" = /etc/dpl-letsencrypt/live/c2.deeplink-subscribe.com ] || exit 0
+nginx -t
+for container in dpl-subscribe-gateway dpl-subscribe-admin; do
+    docker exec "$container" nginx -t
+    docker exec "$container" nginx -s reload
+done
+systemctl reload nginx

renew-node.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+set -eu
+# Copy under renewal-hooks/deploy only after auditing all existing hooks.
+[ "${RENEWED_LINEAGE:-}" = /etc/letsencrypt/live/us01.deeplink-subscribe.com ] || exit 0
+systemctl try-restart dpl-node