dpl/tests/node-onboarding/test_runtime.py
hellcat 5c9157f5d7
Some checks are pending
Docker Build and Publish / build (push) Waiting to run
feat: add reusable jump-host node onboarding runtime
2026-10-17 15:22:30 +08:00

313 lines
23 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Offline adversarial evidence/transport/resume tests; no network or Laravel env."""
import copy
import hashlib
import importlib.util
import json
from pathlib import Path
import sys
import tempfile
import threading
import time
import unittest
from unittest.mock import patch
SCRIPTS=Path(__file__).resolve().parents[2]/'scripts/node-onboarding'
sys.path.insert(0,str(SCRIPTS))
from acceptance import audit,digest,snapshot,service_change,provider_state
from ssh_pool import Pool,private_json,credentials
import batch
class EvidenceTests(unittest.TestCase):
def fixture(self,count):
nodes=[dict(id=i,type='anytls',tag='node-'+str(i),relay='relay.example.invalid',relay_port=47000+i,sni=f'n{i}.example.invalid',expected_exit=f'192.0.2.{i}',country='JP') for i in range(1,count+1)]
report={'state':'live_verified','verified_at':1000,'results':[dict(tag=n['tag'],relay=n['relay'],relay_port=n['relay_port'],sni=n['sni'],exit_ip=n['expected_exit'],country=n['country'],live=True,strict_tls=True) for n in nodes]}
qas={'captured_at':1000,'plan_sha256':'plan','users':{'new':dict(id=9,http_status=403,banned=True,excluded=True)}}
return nodes,report,qas
def check(self,nodes,report,qas,after=None,**bindings):
raw=json.dumps(report).encode();qraw=json.dumps(qas).encode()
binding=dict(plan_sha256='plan',snapshot_sha256=digest(snapshot(nodes)),report_sha256=hashlib.sha256(raw).hexdigest(),qa_rejections_sha256=hashlib.sha256(qraw).hexdigest(),verified_at=1000)
binding.update(bindings)
return audit(nodes,nodes if after is None else after,raw,qraw,binding,'plan',{'new':9},now=1100)
def test_arbitrary_node_counts_and_reordering(self):
for count in [1,2,7,23]:
nodes,report,qas=self.fixture(count)
self.assertEqual(self.check(nodes,report,qas,after=list(reversed(nodes)))['nodes'],count)
def test_topology_change_including_old_node_is_blocked(self):
nodes,r,q=self.fixture(7)
for after in [nodes[:-1],nodes+self.fixture(8)[0][-1:],list(reversed(nodes[:-1]))]:
with self.assertRaises(ValueError):self.check(nodes,r,q,after=after)
altered=copy.deepcopy(nodes);altered[0]['expected_exit']='198.51.100.9'
with self.assertRaises(ValueError):self.check(nodes,r,q,after=altered)
def test_missing_extra_duplicate_or_insecure_results_fail(self):
nodes,r,q=self.fixture(7)
for results in [r['results'][:-1],r['results']+[r['results'][0]],r['results'][1:]+[r['results'][1]]]:
with self.assertRaises(ValueError):self.check(nodes,dict(r,results=results),q)
for field,value in [('exit_ip','198.51.100.1'),('country','US'),('strict_tls',False),('live',False),('sni','wrong.example.invalid')]:
wrong=copy.deepcopy(r);wrong['results'][0][field]=value
with self.assertRaises(ValueError):self.check(nodes,wrong,q)
def test_qa_identity_status_revocation_and_hashes_are_bound(self):
nodes,r,q=self.fixture(2)
for field,value in [('id',10),('http_status',200),('banned',False),('excluded',False)]:
wrong=copy.deepcopy(q);wrong['users']['new'][field]=value
with self.assertRaises(ValueError):self.check(nodes,r,wrong)
for field in ['snapshot_sha256','plan_sha256','report_sha256','qa_rejections_sha256']:
with self.assertRaises(ValueError):self.check(nodes,r,q,**{field:'wrong'})
for stamp in [-1,1200,None,'1000']:
with self.assertRaises(ValueError):self.check(nodes,dict(r,verified_at=stamp),q)
with self.assertRaises(ValueError):self.check(nodes,r,dict(q,captured_at=stamp))
def test_provider_observations_are_exact_and_not_language_or_substring_dependent(self):
node=dict(key='new',tls_domain='a.example.invalid',relay_host='relay.example.invalid',relay_port=47001,node_address='192.0.2.1',listen_port=24443)
observed={'captured_at':1000,'dns':[{'type':'A','name':node['tls_domain'],'address':node['node_address'],'proxy':False}],'relays':[{'name':'dpl-new-anytls','host':node['relay_host'],'port':47001,'target':node['node_address'],'listen_port':24443,'proxy_protocol_in':False,'proxy_protocol_out':False}]}
self.assertTrue(provider_state(observed,node,1100)['dns'])
for section,field,value in [('dns','proxy',True),('dns','address','192.0.2.11'),('relays','proxy_protocol_in',True),('relays','target','192.0.2.11')]:
wrong=copy.deepcopy(observed);wrong[section][0][field]=value
with self.assertRaises(ValueError):provider_state(wrong,node,1100)
self.assertEqual(provider_state(dict(observed,dns=[],relays=[]),node,1100),{'dns':[],'relays':[]})
with self.assertRaises(ValueError):provider_state(observed,node,2000)
def test_registry_and_service_changes_do_not_use_pid_alone(self):
for values in [[],[dict(self.fixture(1)[0][0],type='vless')],self.fixture(1)[0]*2]:
with self.assertRaises(ValueError):snapshot(values)
before=dict(unit_sha256='a',binary_sha256='b',config_sha256='c',listeners=['*:443'],active=True,pid=1,started='yesterday',restarts=0)
self.assertEqual(service_change(before,before),'ready')
self.assertEqual(service_change(before,dict(before,pid=2)),'investigate_restart')
self.assertEqual(service_change(before,dict(before,config_sha256='changed')),'conflict')
self.assertEqual(service_change(before,dict(before,active=False)),'conflict')
self.assertEqual(service_change(before,dict(before,binary_sha256=None)),'unknown')
class TransportTests(unittest.TestCase):
def test_reuse_and_every_nonwhite_host_has_jump(self):
calls=[]
class Client:
active=True
def get_transport(self):return self
def is_active(self):return self.active
def close(self):self.active=False
def factory(rid,spec,jump):calls.append((rid,jump));return Client()
config={'hosts':{'white':{'role':'white_jump'},'panel':{},'node':{}}}
pool=Pool(config,'white',factory)
panel=pool.get('panel');self.assertIs(panel,pool.get('panel'))
node=pool.get('node');self.assertIs(calls[1][1],calls[2][1]);self.assertIsNone(calls[0][1]);self.assertEqual(len(calls),3)
panel.active=False
with self.assertRaises(RuntimeError):pool.get('panel')
self.assertEqual(len(calls),3) # Never automatically replay after disconnect.
pool.close();self.assertFalse(node.active)
def test_sessions_are_not_shared_unsafely_between_workers(self):
barrier=threading.Barrier(2);made=[]
class Client:
def get_transport(self):return self
def is_active(self):return True
def close(self):pass
pool=Pool({'hosts':{'w':{'role':'white_jump'},'n':{}}},'w',lambda *args:Client())
def worker():
n=pool.get('n');barrier.wait();made.append(n);self.assertIs(n,pool.get('n'))
threads=[threading.Thread(target=worker) for _ in range(2)]
for t in threads:t.start()
for t in threads:t.join(timeout=5)
self.assertEqual(len(made),2);self.assertIsNot(made[0],made[1]);pool.close()
def test_existing_codocs_credentials_are_read_without_copy_or_rewrite(self):
with tempfile.TemporaryDirectory() as d:
p=Path(d)/'vps.md';body='# Fixture\n密码:test-only-phrase\n';p.write_text(body);p.chmod(0o600)
self.assertEqual(credentials({'credential_ref':str(p),'credential_format':'codocs-password'}),{'password':'test-only-phrase'})
self.assertEqual(p.read_text(),body);self.assertEqual(len(list(Path(d).iterdir())),1)
p.write_text(body+'密码:ambiguous\n')
with self.assertRaises(ValueError):credentials({'credential_ref':str(p),'credential_format':'codocs-password'})
def test_private_config_and_explicit_jump_enforced(self):
with self.assertRaises(ValueError):Pool({'hosts':{'n':{}}},'w')
with self.assertRaises(ValueError):Pool({'hosts':{'w':{}}},'w')
with tempfile.TemporaryDirectory() as d:
p=Path(d)/'auth.json';p.write_text('{}');p.chmod(0o644)
with self.assertRaises(ValueError):private_json(p)
p.chmod(0o600);self.assertEqual(private_json(p),{})
link=Path(d)/'link';link.symlink_to(p)
with self.assertRaises(ValueError):private_json(link)
class ResumeTests(unittest.TestCase):
def setUp(self):
batch.STOP.clear();self.tmp=tempfile.TemporaryDirectory();self.root=Path(self.tmp.name)
spec=importlib.util.spec_from_file_location('batch_fixtures',Path(__file__).with_name('test_batch.py'));m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
fixture=m.BatchTests();fixture.setUp();self.fixture=fixture;self.plan=fixture.plan
self.calls=[];self.generation='a';self.finished=False
owner=self
class Runner(batch.Coordinator):
def call(self,n,phase,action):
owner.calls.append((phase,action))
r=dict(key=n['key'],phase=phase,request_sha256='irrelevant-in-unit-fixture',status='waiting' if phase=='audit' and not owner.finished else 'ready',valid_until=time.time()+180)
if phase=='preflight':r['resume_dependencies']={k:hashlib.sha256((owner.generation if k=='provision' else k).encode()).hexdigest() for k in ['stage','dns','provision','relay','verify']}
return r
self.Runner=Runner
def tearDown(self):self.fixture.tearDown();self.tmp.cleanup();batch.STOP.clear()
def run_batch(self):return self.Runner(self.plan,self.root/'ledger',jobs=1).run()
def test_fresh_proofs_skip_only_completed_unchanged_phases(self):
self.assertFalse(self.run_batch());self.calls.clear();self.finished=True
self.assertTrue(self.run_batch());self.assertEqual(self.calls,[('preflight','probe'),('publish','probe'),('audit','probe')])
def test_dependency_change_rechecks_downstream_and_not_unrelated_stage(self):
self.run_batch();self.calls.clear();self.generation='b';self.finished=True;self.run_batch()
self.assertEqual([p for p,a in self.calls],['preflight','provision','relay','verify','publish','audit'])
def test_expired_and_unknown_proofs_never_skip(self):
self.run_batch();runner=self.Runner(self.plan,self.root/'ledger',jobs=1)
with runner.ledger.db() as c:c.execute('UPDATE evidence SET expires=0 WHERE phase="dns"')
runner.ledger.mark(self.plan['batch_id'],self.plan['nodes'][0]['key'],'stage','unknown')
self.calls.clear();self.finished=True;self.run_batch()
self.assertIn(('stage','probe'),self.calls);self.assertIn(('dns','probe'),self.calls)
self.assertTrue((self.root/'ledger/events.jsonl').is_file())
class SiteAdapterTests(unittest.TestCase):
def test_preserved_service_review_binds_original_and_exact_observation(self):
from site_adapter import SiteAdapter
adapter=object.__new__(SiteAdapter);adapter.ctx={'preserved_service':dict(unit='XrayR',binary='/bin/xrayr',config='/etc/xrayr.yml',port=443)}
current=dict(unit_sha256='u',binary_sha256='b',config_sha256='c',listeners=['*:443'],active=True,pid=1,started='a',restarts=0)
files={};read=lambda client,path:files.get(path)
def write(client,path,value):files[path]=value.encode() if isinstance(value,str) else value
def execute(client,cmd):return json.dumps(current) if cmd.startswith('python3') else ''
adapter.guard_service(None,'/backup',read,write,execute)
original=copy.deepcopy(current);current.update(pid=2,started='b',restarts=1)
with self.assertRaisesRegex(RuntimeError,'investigate_'):adapter.guard_service(None,'/backup',read,write,execute)
files['/backup/preserved-service-review.json']=json.dumps(dict(before_sha256=digest(original),after_sha256=digest(current),reason='Independently reviewed lifecycle panic',evidence_sha256='e'*64)).encode()
adapter.guard_service(None,'/backup',read,write,execute)
current['config_sha256']='changed'
with self.assertRaisesRegex(RuntimeError,'conflict'):adapter.guard_service(None,'/backup',read,write,execute)
self.assertEqual(json.loads(files['/backup/preserved-service-original.json']),original)
def test_generated_remote_python_and_group_independent_php_compile(self):
import ast
import subprocess
import io
from site_adapter import SiteAdapter
tree=ast.parse((SCRIPTS/'site_adapter.py').read_text())
for method in [n for n in ast.walk(tree) if isinstance(n,ast.FunctionDef) and n.name in ('formal','guard_service','certificate_gate')]:
for c in ast.walk(method):
if isinstance(c,ast.Assign) and any(isinstance(t,ast.Name) and t.id=='script' for t in c.targets):
program=eval(compile(ast.Expression(c.value),'script-builder','eval'),{'repr':repr,'spec':{},'ctx':{'subscription_url':'https://sub.example.invalid'},'value':{'token':'test-only'}})
compile(program,'remote-'+method.name,'exec')
with tempfile.TemporaryDirectory() as d:
a=object.__new__(SiteAdapter);a.plan={'context':dict(panel_vps='panel',gateway_vps='gateway',node_root='/node',ops_root='/ops',app_root='/app',batch_backup_root='/batch',node_backup_root='/backups',qa_root='/qa',gateway_containers=['gateway-a','gateway-b'])};a.ctx=a.plan['context'];a.directory=Path(d)
php_sources=[];calls=[]
class Channel:
def shutdown_write(self):pass
def recv_exit_status(self):return 0
class In:
channel=Channel()
def write(self,body):php_sources.append(body)
class Out:
channel=Channel()
def read(self):
body=php_sources[-1]
if '$base=json_decode' in body:return b'true'
return json.dumps({'manifest':{},'node_online':True,'groups':['7'],'qa_banned':True,'u':1,'d':1}).encode()
class DockerOut:
channel=Channel()
def read(self):return json.dumps([dict(Name='/gateway-a',Id='id-a',State={'StartedAt':'then'},RestartCount=0)]).encode()
class Sftp:
def __enter__(self):return self
def __exit__(self,*args):pass
def open(self,path,mode):return io.BytesIO(json.dumps({'/gateway-a':{'id':'id-a','started':'then','restarts':0}}).encode())
class Remote:
def exec_command(self,cmd,timeout=0):
calls.append(cmd)
if cmd.startswith('docker inspect'):
if cmd!='docker inspect gateway-a gateway-b':raise RuntimeError('Invalid gateway inventory command')
return In(),DockerOut(),io.BytesIO()
return In(),Out(),io.BytesIO()
def open_sftp(self):return Sftp()
class Pool:
def get(self,rid):return Remote()
a.pool=Pool();a.guard_service=lambda *args:None;a.formal=lambda *args:None
n=dict(key='sg-new',request_sha256='a'*64,formal_group_id=7,vps_id='node')
result=a.call(n,'audit','probe');self.assertEqual(result['status'],'ready');self.assertGreaterEqual(len(php_sources),2)
for php in php_sources:
result=subprocess.run(['php','-l'],input=php,text=True,capture_output=True)
self.assertEqual(result.returncode,0,result.stderr)
def test_pool_real_connect_interface_keeps_strict_keys_and_jump_sock(self):
import types
recorded=[]
class FakeClient:
def load_host_keys(self,path):recorded.append(('keys',path))
def set_missing_host_key_policy(self,p):recorded.append(('reject',p))
def get_host_keys(self):return self
def add(self,*args):recorded.append(('approved-key',args))
def connect(self,host,**kw):recorded.append(('connect',host,kw))
def get_transport(self):return self
def is_active(self):return True
def open_channel(self,kind,target,origin):recorded.append(('jump',kind,target));return 'through-white'
def close(self):pass
class Key:
def asbytes(self):return b'approved-public-key-fixture'
def get_name(self):return 'ssh-ed25519'
class Keys:
def __init__(self,path):pass
def lookup(self,name):return {'ssh-ed25519':Key()}
module=types.SimpleNamespace(SSHClient=FakeClient,RejectPolicy=lambda:'reject-unknown',HostKeys=Keys)
pin='SHA256:'+__import__('base64').b64encode(hashlib.sha256(Key().asbytes()).digest()).decode().rstrip('=')
with tempfile.TemporaryDirectory() as d,patch.dict(sys.modules,{'paramiko':module}):
keys=Path(d)/'known_hosts';keys.write_text('public fixture');keys.chmod(0o600)
credentials=Path(d)/'credentials.json';credentials.write_text(json.dumps({'password':'test-only-fixture'}));credentials.chmod(0o600)
hosts={'w':dict(role='white_jump',host='192.0.2.1',username='root',credential_ref=str(credentials)),'n':dict(host='192.0.2.2',username='root',credential_ref=str(credentials))}
for host in hosts.values():host['host_key_sha256']=pin
p=Pool(dict(hosts=hosts,known_hosts=str(keys)),'w');p.get('n');p.get('n');p.close()
before_connects=len([x for x in recorded if x[0]=='connect'])
hosts['w']['host_key_sha256']='SHA256:'+'b'*43
blocked=Pool(dict(hosts=hosts,known_hosts=str(keys)),'w')
with self.assertRaises(ValueError):blocked.get('n')
self.assertEqual(len([x for x in recorded if x[0]=='connect']),before_connects)
blocked.close()
connections=[x for x in recorded if x[0]=='connect'];self.assertEqual(len(connections),2);self.assertIsNone(connections[0][2]['sock']);self.assertEqual(connections[1][2]['sock'],'through-white')
self.assertEqual(len([x for x in recorded if x[0]=='reject']),2)
for _,_,kw in connections:self.assertFalse(kw['allow_agent']);self.assertFalse(kw['look_for_keys'])
def test_configure_is_local_only_and_pins_code_context_and_interpreter(self):
import types
spec=importlib.util.spec_from_file_location('configure_site',SCRIPTS/'configure-site.py');module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
with tempfile.TemporaryDirectory() as d:
root=Path(d);ssh=root/'ssh.json';registry=root/'exits.json';(root/'known_hosts').write_text('public fixture');(root/'known_hosts').chmod(0o600)
hosts={'VPS-001':dict(role='white_jump',host='192.0.2.1',username='root'),'VPS-002':dict(host='192.0.2.2',username='root'),'VPS-003':dict(host='192.0.2.3',username='root'),'VPS-011':dict(host='192.0.2.11',username='root')}
for host in hosts.values():host['host_key_sha256']='SHA256:'+'a'*43
ssh.write_text(json.dumps({'hosts':hosts,'known_hosts':str(root/'known_hosts')}));ssh.chmod(0o600);registry.write_text('{}');registry.chmod(0o600)
fixture_spec=importlib.util.spec_from_file_location('base_fixture',Path(__file__).with_name('test_batch.py'));f=importlib.util.module_from_spec(fixture_spec);fixture_spec.loader.exec_module(f)
original=f.BatchTests();original.setUp()
try:
base=original.plan;node=base['nodes'][0];node['node_address']='192.0.2.11';node['expected_exit']='192.0.2.11';base['context'].update(jump_vps='VPS-001',panel_vps='VPS-002',gateway_vps='VPS-003');registry.write_text(json.dumps({node['tls_domain']:{'expected_exit':node['expected_exit'],'country':node['country']}}))
site=dict(client_sha256='d'*64,ops_root='/ops',node_root='/node',batch_backup_root='/backup/batch',node_backup_root='/backup/nodes',qa_root='/qa/batch',provider_snapshot=str(root/'provider.json'),exit_registry=str(registry),ssh_config=str(ssh),preserved_service=dict(unit='XrayR',binary='/bin/xrayr',config='/etc/xrayr.yml',port=443),formal_socks_port=19999,gateway_containers=['gateway-a'])
with patch.dict(sys.modules,{'paramiko':types.SimpleNamespace(__version__='5.0.0')}):
plan=module.configure(base,site);self.assertEqual(plan['runtime']['kind'],'ssh-site-v1');self.assertFalse((root/'ledger').exists())
runner=batch.Coordinator(plan,root/'ledger',jobs=1);runner.runtime.close();runner.check_runtime()
ssh.write_text(ssh.read_text()+'\n')
with self.assertRaises(ValueError):runner.check_runtime()
finally:original.tearDown()
class PackagingTests(unittest.TestCase):
def test_committed_flat_snapshot_preserves_worktree_index_and_base(self):
import subprocess
spec=importlib.util.spec_from_file_location('bundle_ops',SCRIPTS/'bundle-ops.py');module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
with tempfile.TemporaryDirectory() as d:
root=Path(d);repo=root/'repo';repo.mkdir()
def git(*args):return subprocess.check_output(['git','-C',str(repo),*args],stderr=subprocess.DEVNULL)
git('init');git('config','user.name','Offline Test');git('config','user.email','test@example.invalid');git('config','commit.gpgsign','false')
(repo/'base-only.txt').write_text('preserved operations base');git('add','.');git('commit','-m','base');base=git('rev-parse','HEAD').decode().strip()
source_dir=repo/'scripts/node-onboarding';source_dir.mkdir(parents=True)
for name in module.FILES:(source_dir/name).write_text('committed '+name)
git('add','.');git('commit','-m','source');source=git('rev-parse','HEAD').decode().strip()
(repo/'base-only.txt').write_text('unrelated user modification');(repo/'unrelated.txt').write_text('staged unrelated');git('add','unrelated.txt')
index_before=git('diff','--cached','--binary');status_before=git('status','--porcelain')
output=root/'operations.bundle';result=module.build(repo,source,base,'codex/offline-ops',output)
self.assertTrue(output.exists());self.assertEqual(git('diff','--cached','--binary'),index_before);self.assertEqual(git('status','--porcelain'),status_before)
self.assertEqual(git('show',result['commit']+':panel-acceptance.php'),b'committed panel-acceptance.php')
self.assertEqual(git('show',result['commit']+':base-only.txt'),b'preserved operations base')
self.assertEqual(git('rev-parse','HEAD').decode().strip(),source)
git('bundle','verify',str(output))
with self.assertRaises(ValueError):module.build(repo,source,base,'codex/offline-ops',root/'second.bundle')
def test_event_stream_preserves_final_result_and_avoids_status_polling(self):
import subprocess
spec=importlib.util.spec_from_file_location('event_fixture',Path(__file__).with_name('test_batch.py'));f=importlib.util.module_from_spec(spec);spec.loader.exec_module(f)
test=f.BatchTests();test.setUp()
try:
result=subprocess.run(test.argv()+['--events'],capture_output=True,text=True,timeout=30)
self.assertEqual(result.returncode,0,result.stderr)
lines=[json.loads(x) for x in result.stdout.splitlines()]
self.assertTrue(lines[-1]['complete']);self.assertTrue(any(x.get('phase')=='provision' for x in lines[:-1]))
for x in lines[:-1]:self.assertEqual(set(x),{'batch','node','phase','status','time'})
finally:test.tearDown()
if __name__=='__main__':unittest.main()