441 lines
38 KiB
Python
441 lines
38 KiB
Python
"""Reusable AnyTLS site adapter; invoked in-process by batch.py, never a daemon."""
|
|
import base64,hashlib,json,os,shlex,threading,time
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from ssh_pool import Pool,private_json
|
|
from acceptance import audit,digest,snapshot,service_change,provider_state
|
|
|
|
class SiteAdapter:
|
|
def __init__(self,plan,directory,pool=None,atomic_writer=None,resource_lock=None):
|
|
if atomic_writer is None or resource_lock is None:
|
|
from batch import atomic_json,lock
|
|
atomic_writer,resource_lock=atomic_writer or atomic_json,resource_lock or lock
|
|
self.atomic_json,self.resource_lock=atomic_writer,resource_lock
|
|
self.plan,self.directory=plan,Path(directory)
|
|
self.ctx=plan['context'];self.lock=threading.RLock();self.leases={};self.formal_done={}
|
|
for field in ('ops_root','node_root','batch_backup_root','node_backup_root','qa_root','provider_snapshot','exit_registry','ssh_config','client_binary'):
|
|
value=self.ctx.get(field,'')
|
|
if not isinstance(value,str) or not value.startswith('/') or '..' in value.split('/') or any(c not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_./-' for c in value):
|
|
raise ValueError('Explicit safe absolute site path required: '+field)
|
|
config=private_json(self.ctx['ssh_config'])
|
|
for candidate in plan['nodes']:
|
|
if config['hosts'].get(candidate['vps_id'],{}).get('host')!=candidate['node_address']:raise ValueError('SSH target differs from frozen candidate address')
|
|
for role in ('jump_vps','panel_vps','gateway_vps'):
|
|
if self.ctx[role] not in config['hosts']:raise ValueError('Missing registered SSH role')
|
|
self.pool=pool or Pool(config,self.ctx['jump_vps'])
|
|
if not __import__('re').fullmatch('[0-9a-f]{64}',self.ctx.get('client_sha256','')):raise ValueError('Reviewed official client SHA256 required')
|
|
self.exit_registry=private_json(self.ctx['exit_registry'])
|
|
for candidate in plan['nodes']:
|
|
if self.exit_registry.get(candidate['tls_domain'])!={'expected_exit':candidate['expected_exit'],'country':candidate['country']}:raise ValueError('Candidate exit registry missing or changed')
|
|
if any(n['vps_id'] in [self.ctx['jump_vps'],self.ctx['panel_vps'],self.ctx['gateway_vps']] for n in plan['nodes']):raise ValueError('Dedicated node host required')
|
|
for key in ('app_root',):
|
|
if not __import__('re').fullmatch('/[A-Za-z0-9_./-]+',self.ctx[key]) or '..' in self.ctx[key].split('/'):raise ValueError('Unsafe application root')
|
|
if type(self.ctx.get('formal_socks_port')) is not int or not 1024<=self.ctx['formal_socks_port']<=65535 or self.ctx['formal_socks_port'] in [n['socks_port'] for n in plan['nodes']]:raise ValueError('Reserve a distinct formal SOCKS port')
|
|
if not self.ctx.get('gateway_containers') or any(not __import__('re').fullmatch('[A-Za-z0-9_.-]+',v) for v in self.ctx['gateway_containers']):raise ValueError('Explicit gateway container inventory required')
|
|
|
|
def close(self):
|
|
# On waiting, failure and success, close owned formal leases before closing SSH.
|
|
failed=False
|
|
for group in list(self.leases):
|
|
try:self.account_command(group,'cleanup',True)
|
|
except Exception:failed=True
|
|
self.pool.close()
|
|
if failed:
|
|
self.atomic_json(self.directory/'cleanup-error.json',{'groups':list(self.leases),'reason':'cleanup_outcome_unknown','needs_probe':True})
|
|
raise RuntimeError('Acceptance cleanup incomplete; inspect protected leases before retry')
|
|
|
|
def account_command(self,group,action,apply=False,client=None):
|
|
panel=client or self.pool.get(self.ctx['panel_vps'])
|
|
directory=self.ctx['batch_backup_root']+'/acceptance-group-'+str(group)
|
|
cmd='cd '+self.ctx['app_root']+' && php '+self.ctx['ops_root']+'/panel-acceptance.php --action='+action+' --dir='+directory+' --group='+str(group)+(' --apply' if apply else '')
|
|
_,out,err=panel.exec_command(cmd,timeout=90);data=out.read();err.read()
|
|
if out.channel.recv_exit_status():raise RuntimeError('acceptance_account_blocked')
|
|
try:return json.loads(data)
|
|
except ValueError:raise RuntimeError('acceptance_account_response_not_json') from None
|
|
|
|
def prepare_account(self,n,panel,execute):
|
|
with self.lock:
|
|
group=n['formal_group_id']
|
|
for spec in self.plan['nodes']:
|
|
registered=self.exit_registry.get(spec['tls_domain'],{})
|
|
if registered != {'expected_exit':spec['expected_exit'],'country':spec['country']}:
|
|
raise ValueError('Exit registry must independently cover the candidate plan')
|
|
if group not in self.leases:
|
|
# Register cleanup obligation before any possibly successful remote create.
|
|
self.leases[group]=True
|
|
self.account_command(group,'prepare',True,panel)
|
|
source=self.account_command(group,'source',client=panel)
|
|
for spec in source['nodes']:
|
|
if spec['sni'] not in self.exit_registry:raise ValueError('Missing old-node exit expectation')
|
|
|
|
def certificate_gate(self,n,node,read,execute):
|
|
owner=read(node,'/etc/dpl-node/batch-owner.json')
|
|
owned=bool(owner and json.loads(owner).get('request_sha256')==n['request_sha256'])
|
|
script=r"""from pathlib import Path
|
|
import re
|
|
for name in ['/etc/letsencrypt/cli.ini','/root/.config/letsencrypt/cli.ini']:
|
|
p=Path(name)
|
|
if p.exists():
|
|
for line in p.read_text().splitlines():
|
|
if re.match(r'^\s*(pre[-_]hook|post[-_]hook|deploy[-_]hook)\s*=\s*\S',line):raise SystemExit(2)
|
|
domain=DOMAIN
|
|
paths=[Path('/etc/letsencrypt')/kind/domain for kind in ['live','archive']]+[Path('/etc/letsencrypt/renewal')/(domain+'.conf')]
|
|
if not OWNED and any(p.exists() or p.is_symlink() for p in paths):raise SystemExit(3)
|
|
p=paths[-1]
|
|
if p.exists():
|
|
for line in p.read_text().splitlines():
|
|
if re.match(r'^\s*(pre_hook|post_hook)\s*=\s*\S',line):raise SystemExit(4)
|
|
if re.match(r'^\s*deploy_hook\s*=',line) and line.split('=',1)[1].strip()!=HOOK:raise SystemExit(5)
|
|
"""
|
|
prefix='DOMAIN='+repr(n['tls_domain'])+';OWNED='+repr(owned)+';HOOK='+repr('/bin/sh '+self.ctx['node_root']+'/renew-node.sh')+'\n'
|
|
execute(node,"python3 - <<'DPLCERTGATE'\n"+prefix+script+"\nDPLCERTGATE")
|
|
execute(node,'command -v git >/dev/null && command -v curl >/dev/null && command -v certbot >/dev/null && command -v ss >/dev/null && command -v sha256sum >/dev/null && command -v openssl >/dev/null')
|
|
|
|
def guard_service(self,node,backup,read,write,execute):
|
|
spec=self.ctx['preserved_service']
|
|
for field in ('unit','binary','config','port'):
|
|
if field not in spec:raise ValueError('Explicit preserved service specification required')
|
|
if not all(isinstance(spec[k],str) and __import__('re').fullmatch('[A-Za-z0-9_./@-]+',spec[k]) for k in ('unit','binary','config')) or type(spec['port']) is not int:
|
|
raise ValueError('Unsafe preserved service specification')
|
|
script="""import hashlib,json,pathlib,subprocess
|
|
s=SPEC
|
|
v=dict(x.split('=',1) for x in subprocess.check_output(['systemctl','show',s['unit'],'-p','MainPID','-p','ActiveEnterTimestamp','-p','NRestarts','-p','ActiveState','-p','FragmentPath'],text=True).splitlines())
|
|
def h(p):return hashlib.sha256(pathlib.Path(p).read_bytes()).hexdigest()
|
|
listeners=subprocess.check_output(['ss','-lntH','( sport = :'+str(s['port'])+' )'],text=True)
|
|
print(json.dumps({'unit_sha256':h(v['FragmentPath']),'binary_sha256':h(s['binary']),'config_sha256':h(s['config']),'listeners':sorted(x.split()[3] for x in listeners.splitlines()),'pid':v['MainPID'],'started':v['ActiveEnterTimestamp'],'restarts':v['NRestarts'],'active':v['ActiveState']=='active'}))
|
|
""".replace('SPEC',repr(spec))
|
|
current=json.loads(execute(node,"python3 - <<'DPLSERVICE'\n"+script+"\nDPLSERVICE"))
|
|
if not current['active'] or not current['listeners']:raise RuntimeError('preserved_service_unavailable')
|
|
execute(node,'install -d -m 700 '+backup)
|
|
p=backup+'/preserved-service-original.json';raw=read(node,p)
|
|
if raw is None:write(node,p,json.dumps(current));return current
|
|
before=json.loads(raw);status=service_change(before,current)
|
|
if status=='investigate_restart':
|
|
write(node,backup+'/preserved-service-observed.json',json.dumps(current))
|
|
review=read(node,backup+'/preserved-service-review.json')
|
|
review=json.loads(review) if review else {}
|
|
if review.get('before_sha256')!=digest(before) or review.get('after_sha256')!=digest(current) or not review.get('reason') or not review.get('evidence_sha256'):
|
|
raise RuntimeError('investigate_preserved_service_restart')
|
|
elif status!='ready':raise RuntimeError('preserved_service_'+status)
|
|
return current
|
|
|
|
def dependencies(self,n,panel,gateway,node,stage_probe,read,execute,gateway_py):
|
|
"""Current dependency reads; a missing observation cannot authorize a skip."""
|
|
state=stage_probe(panel);deps={'stage':digest(state)}
|
|
ui=provider_state(private_json(self.ctx['provider_snapshot']),n)
|
|
try:
|
|
dns=json.loads(gateway_py(gateway,'import socket,json\nprint(json.dumps(sorted(set(x[4][0] for x in socket.getaddrinfo('+repr(n['tls_domain'])+',443,socket.AF_INET)))))'))
|
|
if dns==[n['node_address']]:deps['dns']=digest(dns)
|
|
except Exception:pass
|
|
try:
|
|
live=execute(node,'systemctl is-active dpl-node; git -C '+self.ctx['node_root']+' rev-parse HEAD; sha256sum /etc/dpl-node/config.yml /etc/dpl-node/secret.env /etc/systemd/system/dpl-node.service /etc/letsencrypt/live/'+n['tls_domain']+'/fullchain.pem')
|
|
deps['provision']=digest(live)
|
|
# End-to-end strict TLS at preflight is necessary to reuse a relay observation.
|
|
tls=gateway_py(gateway,'import ssl,socket\nc=ssl.create_default_context()\nwith socket.create_connection(('+repr(n['relay_host'])+','+str(n['relay_port'])+'),10) as s:\n with c.wrap_socket(s,server_hostname='+repr(n['tls_domain'])+') as t:print(t.version())')
|
|
rows=ui['relays']
|
|
deps['relay']=digest([rows,tls])
|
|
report=read(panel,self.ctx['node_backup_root']+'/'+n['key']+'/qa-report.json')
|
|
if report:deps['verify']=digest([state,live,hashlib.sha256(report).hexdigest()])
|
|
except Exception:pass
|
|
return deps
|
|
|
|
def formal(self,n,panel,gateway,php,execute,read,write):
|
|
group=n['formal_group_id'];ctx=self.ctx
|
|
with self.lock, self.resource_lock(self.directory.parent, ctx['gateway_vps']+':formal:'+str(ctx['formal_socks_port'])):
|
|
source=self.account_command(group,'snapshot' if group in self.formal_done else 'source',client=panel)
|
|
def public(source):
|
|
values=[]
|
|
for item in source['nodes']:
|
|
if item['sni'] not in self.exit_registry:raise ValueError('Unregistered current node exit')
|
|
values.append(dict(item,**self.exit_registry[item['sni']]))
|
|
return snapshot(values)
|
|
before=public(source)
|
|
# Do not run final verification while any candidate in this group is still isolated.
|
|
cohort=[x for x in self.plan['nodes'] if x['formal_group_id']==group]
|
|
wanted={(x['relay_host'],x['relay_port'],x['tls_domain']) for x in cohort}
|
|
if not wanted.issubset({(x['relay'],x['relay_port'],x['sni']) for x in before}):raise RuntimeError('formal_group_not_ready')
|
|
if group in self.formal_done:
|
|
prior=self.formal_done[group]
|
|
return audit(prior['before'],before,prior['report'],prior['qas'],prior['binding'],digest(self.plan),prior['expected_qas'])
|
|
root=ctx['qa_root']+'/formal-group-'+str(group)
|
|
execute(gateway,'install -d -m 700 '+root)
|
|
expectations=[]
|
|
for node in before:
|
|
original=next(x for x in source['nodes'] if x['id']==node['id'])
|
|
expectations.append({'tag':node['tag'],'host':node['relay'],'port':node['relay_port'],'sni':node['sni'],'uuid':original['uuid'],'expected_exit':node['expected_exit'],'expected_country':node['country']})
|
|
# Tokens/UUIDs never leave memory except the gateway's private task files.
|
|
write(gateway,root+'/source-private.json',json.dumps({'token':source['token'],'expectations':expectations,'owned':source['owned']}))
|
|
script="""import gzip,json,urllib.request,subprocess,os,time
|
|
from pathlib import Path
|
|
os.umask(0o077);root=Path(ROOT);source=json.loads((root/'source-private.json').read_text())
|
|
r=urllib.request.Request(SUB+'/api/v1/client/subscribe?token='+source['token'],headers={'User-Agent':'sing-box/1.14.3'})
|
|
with urllib.request.urlopen(r,timeout=30) as response:
|
|
data=response.read();headers={k.lower():v for k,v in response.headers.items()}
|
|
if headers.get('content-encoding','').lower()=='gzip':data=gzip.decompress(data)
|
|
assert headers.get('cf-cache-status','').upper() not in ['HIT','STALE','UPDATING'] and not headers.get('age') and not headers.get('cf-mitigated') and 'no-store' in headers.get('cache-control','').lower()
|
|
(root/'subscription.json').write_bytes(data);(root/'expectations.json').write_text(json.dumps(source['expectations']))
|
|
deadline=time.monotonic()+900
|
|
for attempt in range(3 if source['owned'] else 1):
|
|
with (root/'verify.stdout').open('wb') as out,(root/'verify.stderr').open('wb') as err:
|
|
result=subprocess.run(['python3',OPS+'/verify-node.py','--subscription',str(root/'subscription.json'),'--expectations',str(root/'expectations.json'),'--binary',BINARY,'--port',str(PORT),'--report',str(root/'report.json'),'--run'],timeout=max(1,deadline-time.monotonic()),stdout=out,stderr=err)
|
|
if result.returncode==0:break
|
|
try:reason=json.loads((root/'verify.stderr').read_text()).get('reason')
|
|
except Exception:reason=None
|
|
if not source['owned'] or attempt==2 or reason!='Verification failed; inspect protected diagnostics' or deadline-time.monotonic()<15:raise RuntimeError('Formal verification failed')
|
|
# A just-created user's UUID may need one node pull interval to propagate.
|
|
# Retry only diagnostic reads, keep the same lease; never replay production writes.
|
|
time.sleep(15)
|
|
"""
|
|
prefix='ROOT='+repr(root)+';SUB='+repr(ctx['subscription_url'])+';OPS='+repr(ctx['ops_root'])+';BINARY='+repr(ctx['client_binary'])+';PORT='+repr(ctx['formal_socks_port'])+'\n'
|
|
try:
|
|
execute(gateway,"python3 - <<'DPLFORMAL'\n"+prefix+script+"\nDPLFORMAL",1000)
|
|
after=public(self.account_command(group,'source',client=panel))
|
|
report=read(gateway,root+'/report.json')
|
|
finally:
|
|
# Never leave an owned formal test user active, including failed client runs.
|
|
self.account_command(group,'cleanup',True,panel)
|
|
identities={};tokens={}
|
|
for candidate in cohort:
|
|
b=ctx['node_backup_root']+'/'+candidate['key']
|
|
result=php(panel,'$m=json_decode(file_get_contents('+self.php_value(b+'/manifest.json')+'),true);$s=App\\Models\\Server::findOrFail($m["node_id"]);$u=App\\Models\\User::findOrFail($m["test_user_id"]);echo json_encode(["id"=>$u->id,"token"=>$u->token,"banned"=>(bool)$u->banned,"excluded"=>!App\\Services\\ServerService::getAvailableUsers($s)->contains("id",$u->id)]);')
|
|
identities[candidate['key']]=result['id'];tokens[candidate['key']]=result
|
|
# Include every owned formal lease from earlier resumptions, not just the current user.
|
|
lease_dir=ctx['batch_backup_root']+'/acceptance-group-'+str(group)
|
|
leases=php(panel,'$out=[];foreach(glob('+self.php_value(lease_dir+'/acceptance-*.json')+') as $f){$j=json_decode(file_get_contents($f),true);if(!($j["owned"]??false)||empty($j["id"]))continue;$u=App\\Models\\User::findOrFail($j["id"]);$excluded=true;foreach(App\\Models\\Server::all() as $s)if(App\\Services\\ServerService::getAvailableUsers($s)->contains("id",$u->id))$excluded=false;$out[(string)$u->id]=["id"=>$u->id,"token"=>$u->token,"banned"=>(bool)$u->banned,"excluded"=>$excluded];}echo json_encode($out);')
|
|
for ident,value in (leases.items() if isinstance(leases,dict) else []):
|
|
key='formal-'+str(ident);identities[key]=value['id'];tokens[key]=value
|
|
users={}
|
|
for key,value in tokens.items():
|
|
# Only the gateway accesses the subscription source; local HTTP never bypasses its registered route.
|
|
script='import urllib.request,urllib.error\nr=urllib.request.Request('+repr(ctx['subscription_url']+'/api/v1/client/subscribe?token='+value['token'])+',headers={"User-Agent":"sing-box/1.14.3"})\ntry:\n response=urllib.request.urlopen(r,timeout=30);print(response.status);response.close()\nexcept urllib.error.HTTPError as e:print(e.code)'
|
|
# Keep tokens out of argv: exec_command carries a private stdin script, not an OS argv.
|
|
si,o,e=gateway.exec_command('python3 -',timeout=60);si.write(script);si.channel.shutdown_write();out=o.read();e.read()
|
|
if o.channel.recv_exit_status():raise RuntimeError('qa_rejection_probe_failed')
|
|
users[key]={'id':value['id'],'banned':value['banned'],'excluded':value['excluded'],'http_status':int(out.strip())}
|
|
qas=json.dumps({'users':users,'captured_at':int(time.time()),'plan_sha256':digest(self.plan)},sort_keys=True).encode()
|
|
binding={'plan_sha256':digest(self.plan),'snapshot_sha256':digest(before),'report_sha256':hashlib.sha256(report).hexdigest(),'qa_rejections_sha256':hashlib.sha256(qas).hexdigest(),'verified_at':int(time.time())}
|
|
audit(before,after,report,qas,binding,digest(self.plan),identities)
|
|
write(gateway,root+'/qa-rejections.json',qas);write(gateway,root+'/binding.json',json.dumps(binding));write(gateway,root+'/snapshot.json',json.dumps(before))
|
|
self.formal_done[group]={'before':before,'report':report,'qas':qas,'binding':binding,'expected_qas':identities}
|
|
# Persist only nonsecret, byte-bound results for coordinator/operator review.
|
|
self.atomic_json(self.directory/('formal-group-'+str(group)+'.json'),{'snapshot':before,'report':json.loads(report),'qas':json.loads(qas),'binding':binding})
|
|
|
|
def qa_evidence(self,n,raw,state):
|
|
# Bind the exact remote bytes, not a re-serialized nested report. Persist the latest
|
|
# online/traffic state so independent review never needs a separate task collector.
|
|
report=json.loads(raw)
|
|
self.atomic_json(self.directory/(n['key']+'-evidence.json'),{
|
|
'key':n['key'],'request_sha256':n['request_sha256'],
|
|
'report_sha256':hashlib.sha256(raw).hexdigest(),'report':report,
|
|
'panel':state,'collected_at':int(time.time())})
|
|
|
|
@staticmethod
|
|
def php_value(value):
|
|
return "json_decode(base64_decode('"+base64.b64encode(json.dumps(value).encode()).decode()+"'),true)"
|
|
|
|
def call(self,n,phase,action):
|
|
plan=self.plan;ctx=self.ctx;key=n['key'];sha=n['request_sha256']
|
|
a=SimpleNamespace(phase=phase,action=action,state_dir=str(self.directory))
|
|
B=ctx['node_backup_root']+'/'+key;Q=ctx['qa_root']+'/'+key
|
|
OPS=ctx['ops_root'];NP=ctx['node_root'];APP=ctx['app_root']
|
|
def execute(c,cmd,timeout=120):
|
|
si,o,e=c.exec_command(cmd,timeout=timeout);out=o.read();err=e.read();code=o.channel.recv_exit_status()
|
|
if code: raise RuntimeError('remote_exit_'+str(code))
|
|
return out.decode()
|
|
def php(c,body):
|
|
si,o,e=c.exec_command('cd '+shlex.quote(APP)+' && php',timeout=90)
|
|
si.write("<?php require 'vendor/autoload.php';$app=require 'bootstrap/app.php';$app->make(Illuminate\\Contracts\\Console\\Kernel::class)->bootstrap();"+body)
|
|
si.channel.shutdown_write();out=o.read();err=e.read()
|
|
if o.channel.recv_exit_status():raise RuntimeError('panel_php_failed')
|
|
try:return json.loads(out)
|
|
except ValueError:raise RuntimeError('panel_response_not_json') from None
|
|
def payload(value):return "json_decode(base64_decode('"+base64.b64encode(json.dumps(value,ensure_ascii=False).encode()).decode()+"'),true)"
|
|
def read(c,path):
|
|
with c.open_sftp() as s:
|
|
try:
|
|
with s.open(path,'rb') as f:return f.read()
|
|
except FileNotFoundError:return None
|
|
except OSError as e:
|
|
if e.errno==2:return None
|
|
raise
|
|
def write(c,path,data):
|
|
if isinstance(data,str):data=data.encode()
|
|
with c.open_sftp() as s:
|
|
tmp=path+'.tmp-'+str(os.getpid())+'-'+str(threading.get_ident())
|
|
with s.open(tmp,'w') as f:s.chmod(tmp,0o600);f.write(data)
|
|
s.posix_rename(tmp,path)
|
|
def parameters():return dict(sorted({'key':key,'name':n['name'],'relay-host':n['relay_host'],'relay-port':n['relay_port'],'listen-port':n['listen_port'],'tls-domain':n['tls_domain'],'panel-url':ctx['panel_url']}.items()))
|
|
def canonical(value):return hashlib.sha256(json.dumps(value,sort_keys=True,ensure_ascii=False,separators=(',',':')).encode()).hexdigest()
|
|
def containers(gateway):
|
|
data=json.loads(execute(gateway,'docker inspect '+ ' '.join(ctx['gateway_containers'])))
|
|
return {v['Name']:{'id':v['Id'],'started':v['State']['StartedAt'],'restarts':v['RestartCount']} for v in data}
|
|
def receipt(status,retry=False):return {'key':key,'phase':a.phase,'request_sha256':sha,'status':status,'valid_until':time.time()+300,**({'retry_safe':True} if retry else {})}
|
|
def stage_probe(panel):
|
|
p=parameters()
|
|
code='$b='+payload(B)+';$p='+payload(p)+r''';$f=$b.'/manifest.json';if(!is_file($f)){$conf=App\Models\ServerMachine::where('name','dpl-'.$p['key'])->exists()||App\Models\Server::where('name',$p['name'])->exists()||App\Models\Server::where('host',$p['relay-host'])->where('port',(string)$p['relay-port'])->exists();echo json_encode(['status'=>$conf?'conflict':'absent','retry_safe'=>!$conf]);return;}$m=json_decode(file_get_contents($f),true);$s=App\Models\Server::find($m['node_id']);$mch=App\Models\ServerMachine::find($m['machine_id']);$ok=($m['parameters']??null)===$p && $s && $mch && $s->machine_id===$mch->id && $mch->name==='dpl-'.$p['key'] && $s->type==='anytls' && ($s->protocol_settings['tls']['allow_insecure']??null)===false && $s->name===$p['name'] && $s->host===$p['relay-host'] && (int)$s->port===$p['relay-port'] && (int)$s->server_port===$p['listen-port'] && ($s->protocol_settings['tls']['server_name']??null)===$p['tls-domain'];$u=App\Models\User::find($m['test_user_id']);$g=App\Models\ServerGroup::find($m['group_id']);$ok=$ok && $u && $g && !$u->is_admin && (int)$u->group_id===$m['group_id'] && str_starts_with($u->email,'node-qa-'.$p['key'].'-') && $g->name==='dpl 接入验收 '.$p['key'];if(!is_file($b.'/promotion-journal.json'))$ok=$ok && $s->group_ids===[(string)$m['group_id']] && !$u->banned;echo json_encode(['status'=>$ok?'ready':'conflict','manifest'=>$m]);'''
|
|
return php(panel,code)
|
|
def state(panel):
|
|
return php(panel,'$b='+payload(B)+r''';$m=json_decode(file_get_contents($b.'/manifest.json'),true);$s=App\Models\Server::findOrFail($m['node_id']);$u=App\Models\User::findOrFail($m['test_user_id']);echo json_encode(['manifest'=>$m,'node_online'=>(bool)$s->is_online,'groups'=>$s->group_ids,'qa_banned'=>(bool)$u->banned,'u'=>(int)$u->u,'d'=>(int)$u->d]);''')
|
|
def baseline(panel,gateway):
|
|
directory=ctx['batch_backup_root'];f=directory+'/baseline.json'
|
|
version=json.loads(execute(panel,"python3 - <<'DPLVERSION'\nimport subprocess,hashlib,json\np="+repr(APP)+"\nh=subprocess.check_output(['git','-C',p,'rev-parse','HEAD']).decode().strip()\nd=subprocess.check_output(['git','-C',p,'diff','HEAD'])+subprocess.check_output(['git','-C',p,'status','--porcelain'])\nprint(json.dumps({'head':h,'worktree_sha256':hashlib.sha256(d).hexdigest()}))\nDPLVERSION"))
|
|
previous=read(panel,directory+'/application-version.json')
|
|
if previous is None:write(panel,directory+'/application-version.json',json.dumps(version))
|
|
elif json.loads(previous)!=version:raise RuntimeError('application_version_changed')
|
|
# Read/write a protected snapshot only once, before any new stage.
|
|
return php(panel,'$p='+payload(f)+r''';if(is_file($p)){echo json_encode(['exists'=>true]);return;} $v=['nodes'=>App\Models\Server::orderBy('id')->get()->toArray(),'plans'=>App\Models\Plan::orderBy('id')->get()->toArray(),'users'=>App\Models\User::orderBy('id')->get(['id','group_id','plan_id'])->toArray(),'subscribe_url'=>admin_setting('subscribe_url')];file_put_contents($p,json_encode($v,JSON_THROW_ON_ERROR));chmod($p,0600);echo json_encode(['exists'=>true]);''')
|
|
def gateway_py(gateway,script,timeout=120):
|
|
return execute(gateway,"python3 - <<'DPLPY'\n"+script+"\nDPLPY",timeout)
|
|
def assert_old(node):
|
|
self.guard_service(node,B,read,write,execute)
|
|
def transfer(panel,gateway,node):
|
|
execute(node,'install -d -m 700 /etc/dpl-node')
|
|
execute(gateway,'install -d -m 700 '+Q)
|
|
for source,dest,target in [('node-config.yml',node,'/etc/dpl-node/config.yml'),('node-secret.env',node,'/etc/dpl-node/secret.env'),('qa-private.json',gateway,Q+'/qa.json')]:
|
|
data=read(panel,B+'/'+source);prior=read(dest,target)
|
|
if prior is not None and prior!=data:raise RuntimeError('private_config_conflict')
|
|
if prior is None:write(dest,target,data)
|
|
def fetch_qa(gateway):
|
|
script='''import json,urllib.request
|
|
+from pathlib import Path
|
|
+q=json.loads(Path(Q+'/qa.json').read_text())
|
|
+req=urllib.request.Request(SUB+'/api/v1/client/subscribe?token='+q['subscribe_token'],headers={'User-Agent':'sing-box/1.14.3'})
|
|
+with urllib.request.urlopen(req,timeout=30) as r:
|
|
+ data=r.read(); code=r.status; headers={k.lower():v for k,v in r.headers.items()}
|
|
+import gzip
|
|
+if headers.get('content-encoding','').lower()=='gzip':data=gzip.decompress(data)
|
|
+v=json.loads(data);Path(Q+'/subscription.json').write_bytes(data);Path(Q+'/subscription.json').chmod(0o600)
|
|
+print(json.dumps({'status':code,'cache':headers.get('cf-cache-status'),'control':headers.get('cache-control'),'age':headers.get('age')}))
|
|
+'''.replace('\n+','\n')
|
|
return gateway_py(gateway,'Q='+repr(Q)+';SUB='+repr(ctx['subscription_url'])+'\n'+script)
|
|
def phase_impl(panel,gateway,node):
|
|
if a.phase=='preflight':
|
|
with self.lock, self.resource_lock(self.directory.parent,ctx['panel_vps']+':panel'):
|
|
execute(panel,'install -d -m 700 '+ctx['batch_backup_root'])
|
|
execute(panel,'test "$(git -C '+OPS+' rev-parse HEAD)" = '+ctx['provision_commit']+' && test -z "$(git -C '+OPS+' status --porcelain)"')
|
|
baseline(panel,gateway)
|
|
base_path=ctx['batch_backup_root']+'/gateway-baseline.json'
|
|
current_containers=containers(gateway)
|
|
prior_gateway=read(panel,base_path)
|
|
if prior_gateway is None:write(panel,base_path,json.dumps(current_containers,sort_keys=True))
|
|
elif json.loads(prior_gateway)!=current_containers:return receipt('conflict')
|
|
ui=Path(ctx['provider_snapshot'])
|
|
if not ui.exists():return receipt('waiting')
|
|
snap=private_json(ui)
|
|
if type(snap.get('captured_at')) not in (int,float) or not 0<=time.time()-snap['captured_at']<=900:return receipt('waiting')
|
|
try:provider_state(snap,n)
|
|
except ValueError:return receipt('conflict')
|
|
gateway_py(gateway,'import socket\ns=socket.socket();s.bind(("127.0.0.1",'+str(n['socks_port'])+'));s.close()')
|
|
s=stage_probe(panel)
|
|
if s['status']=='conflict':return receipt('conflict')
|
|
assert_old(node)
|
|
self.certificate_gate(n,node,read,execute)
|
|
execute(node,'test "$(git -C '+NP+' rev-parse HEAD)" = '+ctx['provision_commit']+' && test -z "$(git -C '+NP+' status --porcelain)" && command -v git >/dev/null')
|
|
execute(gateway,"echo '"+ctx['client_sha256']+" "+ctx['client_binary']+"' | sha256sum -c --status && test -x "+shlex.quote(ctx['client_binary'])+' && test "$(git -C '+OPS+' rev-parse HEAD)" = '+ctx['provision_commit']+' && test -z "$(git -C '+OPS+' status --porcelain)"')
|
|
if s['status']=='absent':execute(node,"test ! -e /etc/dpl-node/config.yml && test ! -e /etc/systemd/system/dpl-node.service && test ! -e /etc/dpl-node/batch-owner.json && ! ss -lntH '( sport = :"+str(n['listen_port'])+" )' | grep -q .")
|
|
if not php(panel,'echo json_encode(App\\Models\\ServerGroup::where("id",'+str(n['formal_group_id'])+')->exists());'):return receipt('conflict')
|
|
self.prepare_account(n,panel,execute)
|
|
r=receipt('ready');r['resume_dependencies']=self.dependencies(n,panel,gateway,node,stage_probe,read,execute,gateway_py);return r
|
|
if a.phase=='stage':
|
|
if a.action=='probe':
|
|
r=stage_probe(panel);return receipt(r['status'],r.get('retry_safe',False))
|
|
command='cd '+APP+' && php '+OPS+'/stage.php '+ ' '.join('--'+k+'='+shlex.quote(str(v)) for k,v in {**parameters(),'backup':B}.items())
|
|
execute(panel,command);return receipt('ready')
|
|
if a.phase=='dns':
|
|
try:
|
|
result=gateway_py(gateway,'import socket,json\nprint(json.dumps(sorted(set(x[4][0] for x in socket.getaddrinfo('+repr(n['tls_domain'])+',443,socket.AF_INET)))))')
|
|
return receipt('ready' if json.loads(result)==[n['node_address']] else 'conflict')
|
|
except Exception:return receipt('waiting')
|
|
if a.phase=='provision':
|
|
owner='/etc/dpl-node/batch-owner.json';prior=read(node,owner)
|
|
if prior is not None and json.loads(prior).get('request_sha256')!=sha:return receipt('conflict')
|
|
if prior is None and read(node,'/etc/systemd/system/dpl-node.service') is not None:return receipt('conflict')
|
|
done=read(node,B+'/provision-ok.json')
|
|
if a.action=='probe':
|
|
if done and json.loads(done).get('request_sha256')==sha:
|
|
for file,target in [('node-config.yml','/etc/dpl-node/config.yml'),('node-secret.env','/etc/dpl-node/secret.env')]:
|
|
if read(panel,B+'/'+file)!=read(node,target):return receipt('conflict')
|
|
execute(node,"echo '55bf71fa9d9f2048d3255ae7c0af929a41897ca7743f6ead34132a6ca4c79043 /opt/dpl-node/xboard-node' | sha256sum -c >/dev/null")
|
|
assert_old(node);execute(node,'systemctl is-active --quiet dpl-node && openssl x509 -checkend 86400 -noout -in /etc/letsencrypt/live/'+n['tls_domain']+'/fullchain.pem')
|
|
return receipt('ready')
|
|
return receipt('absent',True)
|
|
if prior is None:
|
|
execute(node,'install -d -m 700 /etc/dpl-node');write(node,owner,json.dumps({'request_sha256':sha,'key':key}))
|
|
transfer(panel,gateway,node)
|
|
if read(node,'/etc/systemd/system/dpl-node.service') is None:
|
|
execute(node,'bash '+NP+'/install-node.sh --listen-port='+str(n['listen_port'])+' --tls-domain='+n['tls_domain']+' > '+B+'/install-node.log 2>&1',300)
|
|
execute(node,"echo '55bf71fa9d9f2048d3255ae7c0af929a41897ca7743f6ead34132a6ca4c79043 /opt/dpl-node/xboard-node' | sha256sum -c >/dev/null")
|
|
domain=n['tls_domain']
|
|
if read(node,'/etc/letsencrypt/live/'+domain+'/fullchain.pem') is None:
|
|
execute(node,"certbot certonly --standalone --non-interactive --agree-tos --no-directory-hooks --cert-name "+domain+' -d '+domain+" --deploy-hook '/bin/sh "+NP+"/renew-node.sh' > "+B+'/certbot.log 2>&1',300)
|
|
execute(node,'systemctl enable --now dpl-node > '+B+'/enable.log 2>&1 && systemctl is-active --quiet dpl-node',60)
|
|
execute(node,'certbot renew --cert-name '+domain+' --dry-run --no-random-sleep-on-renew --run-deploy-hooks --no-directory-hooks > '+B+'/renew-dry-run.log 2>&1',600)
|
|
assert_old(node)
|
|
write(node,B+'/provision-ok.json',json.dumps({'request_sha256':sha,'verified_at':int(time.time())}))
|
|
return receipt('ready')
|
|
if a.phase=='relay':
|
|
try:
|
|
result=gateway_py(gateway,'import ssl,socket,json\nc=ssl.create_default_context()\nwith socket.create_connection(('+repr(n['relay_host'])+','+str(n['relay_port'])+'),10) as s:\n with c.wrap_socket(s,server_hostname='+repr(n['tls_domain'])+') as t: print(json.dumps({"tls":t.version()}))')
|
|
return receipt('ready' if json.loads(result).get('tls') else 'waiting')
|
|
except Exception:return receipt('waiting')
|
|
if a.phase=='verify':
|
|
s=state(panel)
|
|
if s['qa_banned'] and s['groups']==[str(n['formal_group_id'])]:
|
|
return receipt('ready' if read(panel,B+'/qa-report.json') else 'unknown')
|
|
existing=read(panel,B+'/qa-report.json')
|
|
if existing:
|
|
report=json.loads(existing)
|
|
if report.get('state')=='live_verified' and report.get('qa_user_id')==s['manifest']['test_user_id'] and 0<=time.time()-report.get('verified_at',0)<=900 and s['node_online'] and s['u']>0 and s['d']>0:
|
|
self.qa_evidence(n,existing,s)
|
|
r=receipt('ready');r['valid_until']=min(r['valid_until'],report['verified_at']+900);return r
|
|
transfer(panel,gateway,node);fetch_qa(gateway)
|
|
execute(gateway,'python3 '+OPS+'/verify-node.py --qa '+Q+'/qa.json --subscription '+Q+'/subscription.json --binary '+ctx['client_binary']+' --port '+str(n['socks_port'])+' --expected-exit '+shlex.quote(n['expected_exit'])+' --expected-country '+n['country']+' --report '+Q+'/qa-report.json --run > '+Q+'/verify.stdout 2> '+Q+'/verify.stderr',300)
|
|
data=read(gateway,Q+'/qa-report.json');report=json.loads(data)
|
|
write(panel,B+'/qa-report.json',data)
|
|
for _ in range(24):
|
|
s=state(panel)
|
|
if s['node_online'] and s['u']>0 and s['d']>0:
|
|
self.qa_evidence(n,data,s)
|
|
r=receipt('ready');r['valid_until']=min(r['valid_until'],report['verified_at']+900);return r
|
|
time.sleep(5)
|
|
self.qa_evidence(n,data,s)
|
|
return receipt('waiting')
|
|
if a.phase=='publish':
|
|
s=state(panel)
|
|
report_data=read(panel,B+'/qa-report.json')
|
|
review_path=Path(a.state_dir)/(key+'-reviewed.json')
|
|
if not report_data or not review_path.exists():return receipt('waiting')
|
|
report=json.loads(report_data);review=json.loads(review_path.read_text())
|
|
valid=review.get('request_sha256')==sha and review.get('report_sha256')==hashlib.sha256(report_data).hexdigest() and review.get('qa_user_id')==s['manifest']['test_user_id'] and review.get('verified_at')==report.get('verified_at')
|
|
if not valid:return receipt('waiting')
|
|
if s['qa_banned'] and s['groups']==[str(n['formal_group_id'])]:
|
|
journal=read(panel,B+'/promotion-journal.json')
|
|
if not journal:return receipt('conflict')
|
|
j=json.loads(journal)
|
|
return receipt('ready' if j.get('manifest')==s['manifest'] and j.get('request',{}).get('mode')=='append' and j.get('request',{}).get('group')==n['formal_group_id'] and j.get('state')=='applied' else 'conflict')
|
|
if not s['node_online'] or s['u']<=0 or s['d']<=0:return receipt('waiting')
|
|
if a.action=='probe':return receipt('absent',True)
|
|
cmd='cd '+APP+' && php '+OPS+'/promote.php --backup='+B+' --mode=append --group='+str(n['formal_group_id'])+' --verification='+B+'/qa-report.json'
|
|
execute(panel,cmd);execute(panel,cmd+' --apply');return receipt('ready')
|
|
if a.phase=='audit':
|
|
s=state(panel);assert_old(node)
|
|
if not s['qa_banned'] or s['groups']!=[str(n['formal_group_id'])] or not s['node_online']:return receipt('conflict')
|
|
ok=php(panel,'$b='+payload(B)+r''';$m=json_decode(file_get_contents($b.'/manifest.json'),true);$n=App\Models\Server::findOrFail($m['node_id']);$base=json_decode(file_get_contents('''+payload(ctx['batch_backup_root']+'/baseline.json')+r'''),true);$ok=$base['plans']===App\Models\Plan::orderBy('id')->get()->toArray() && $base['users']===App\Models\User::whereIn('id',array_column($base['users'],'id'))->orderBy('id')->get(['id','group_id','plan_id'])->toArray() && $base['subscribe_url']===admin_setting('subscribe_url');foreach($base['nodes'] as $old){$cur=App\Models\Server::find($old['id'])->toArray();foreach(['name','host','port','server_port','protocol_settings','group_ids','machine_id','enabled','show','cert_config'] as $k)if(($cur[$k]??null)!==($old[$k]??null))$ok=false;}$ids=App\Services\ServerService::getAvailableUsers($n)->pluck('id')->all();$ok=$ok&&!in_array($m['test_user_id'],$ids);echo json_encode($ok);''')
|
|
if not ok:return receipt('conflict')
|
|
self.formal(n,panel,gateway,php,execute,read,write)
|
|
original_gateway=read(panel,ctx['batch_backup_root']+'/gateway-baseline.json')
|
|
if not original_gateway or containers(gateway)!=json.loads(original_gateway):return receipt('conflict')
|
|
return receipt('ready')
|
|
return receipt('unknown')
|
|
panel=Lazy(self.pool,ctx['panel_vps']);gateway=Lazy(self.pool,ctx['gateway_vps']);node=Lazy(self.pool,n['vps_id'])
|
|
try:return phase_impl(panel,gateway,node)
|
|
except Exception as exc:
|
|
# Never print underlying library/server errors or credentials.
|
|
safe = str(exc) if type(exc) is RuntimeError and __import__('re').fullmatch('[a-z_0-9]+',str(exc)) else 'inspect_protected_state'
|
|
self.atomic_json(self.directory/(key+'-'+phase+'-error.json'),{'reason':safe})
|
|
return receipt('waiting' if safe in ('investigate_preserved_service_restart','formal_group_not_ready') else 'unknown')
|
|
|
|
class Lazy:
|
|
def __init__(self,pool,rid):self.pool,self.rid=pool,rid
|
|
def __getattr__(self,name):return getattr(self.pool.get(self.rid),name)
|