203 lines
9.6 KiB
Python
203 lines
9.6 KiB
Python
"""Offline process-level tests. No Laravel bootstrap, .env, SSH or network."""
|
|
import copy
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
RUNNER = ROOT / 'scripts/node-onboarding/batch.py'
|
|
ADAPTER = ROOT / 'scripts/node-onboarding/command-adapter.py'
|
|
spec = importlib.util.spec_from_file_location('dpl_batch', RUNNER)
|
|
batch = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(batch)
|
|
|
|
FAKE = r'''
|
|
import argparse, json, os, sys, time
|
|
from pathlib import Path
|
|
p=argparse.ArgumentParser()
|
|
for x in ('plan','node','phase','action','state-dir'): p.add_argument('--'+x,required=True)
|
|
a=p.parse_args(); n=json.loads(Path(a.node).read_text()); plan=json.loads(Path(a.plan).read_text())
|
|
root=Path(plan['fixture_root']); root.mkdir(exist_ok=True)
|
|
identity=dict(key=n['key'], phase=a.phase, request_sha256=n['request_sha256'])
|
|
def emit(status, **kw): print(json.dumps(dict(identity,status=status,**kw)))
|
|
if plan.get('bad_receipt'): identity['request_sha256']='wrong'
|
|
if a.phase=='preflight':
|
|
emit('waiting' if plan.get('block_preflight') and n['key']=='jp2' else 'ready'); sys.exit()
|
|
if a.phase in ('verify','audit'): emit('ready'); sys.exit()
|
|
marker=root/(n['key']+'-'+a.phase)
|
|
if a.action=='probe':
|
|
emit('ready' if marker.exists() else 'absent',retry_safe=True); sys.exit()
|
|
def log(event):
|
|
fd=os.open(root/'events',os.O_CREAT|os.O_WRONLY|os.O_APPEND,0o600)
|
|
os.write(fd,(json.dumps(dict(key=n['key'],phase=a.phase,event=event,t=time.monotonic()))+'\n').encode());os.close(fd)
|
|
log('start'); time.sleep(1.2 if a.phase=='provision' else .12); marker.write_text('owned'); log('end')
|
|
if plan.get('crash') and a.phase=='provision' and not (root/'crashed').exists():
|
|
(root/'crashed').write_text('yes'); sys.exit(8)
|
|
emit('ready')
|
|
'''
|
|
|
|
|
|
class BatchTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.root = Path(self.tmp.name)
|
|
self.fake = self.root / 'fake.py'
|
|
self.fake.write_text(FAKE)
|
|
self.state = self.root / 'state'
|
|
self.plan = {
|
|
'version': 1, 'mode': 'append', 'batch_id': 'offline-test',
|
|
'context': {'panel_vps': 'VPS-001', 'gateway_vps': 'VPS-002',
|
|
'jump_vps': 'VPS-003', 'app_root': '/app',
|
|
'panel_url': 'https://panel.example.invalid',
|
|
'subscription_url': 'https://sub.example.invalid',
|
|
'provision_commit': 'a' * 40, 'bundle': '/private/bundle',
|
|
'client_binary': '/private/sing-box'},
|
|
'adapter': [sys.executable, str(self.fake)],
|
|
'fixture_root': str(self.root / 'remote'), 'nodes': [self.node(1)],
|
|
}
|
|
self.path = self.root / 'plan.json'
|
|
|
|
def tearDown(self):
|
|
self.tmp.cleanup()
|
|
|
|
def node(self, i):
|
|
return dict(key=f'jp{i}', country='JP', sequence=i, flag='🇯🇵', country_name='日本',
|
|
name=f'🇯🇵 日本 JP - {i:02d}', vps_id=f'VPS-{10+i:03d}',
|
|
node_address=f'192.0.2.{i}', listen_port=24443, relay_host='relay.example.invalid',
|
|
relay_port=47000+i, tls_domain=f'jp{i}.example.invalid', formal_group_id=1,
|
|
expected_exit=f'192.0.2.{i}', socks_port=19000+i)
|
|
|
|
def argv(self, command='run'):
|
|
self.path.write_text(json.dumps(self.plan, ensure_ascii=False))
|
|
return [sys.executable, str(RUNNER), command, '--plan', str(self.path),
|
|
'--state-root', str(self.state), '--jobs', '3'] + (['--execute'] if command == 'run' else [])
|
|
|
|
def run_cli(self, command='run', expected=0):
|
|
p = subprocess.run(self.argv(command), capture_output=True, text=True, timeout=30)
|
|
self.assertEqual(p.returncode, expected, p.stdout + p.stderr)
|
|
return json.loads(p.stdout)
|
|
|
|
def events(self):
|
|
path = self.root / 'remote/events'
|
|
return [json.loads(x) for x in path.read_text().splitlines()] if path.exists() else []
|
|
|
|
def test_plan_and_status_are_read_only(self):
|
|
self.run_cli('plan'); self.run_cli('status')
|
|
self.assertFalse(self.state.exists())
|
|
self.assertFalse((self.root/'remote').exists())
|
|
|
|
def test_duplicate_parameters_and_naming_rejected(self):
|
|
for field in ('key','name','vps_id','node_address','tls_domain','relay_port','socks_port','sequence'):
|
|
with self.subTest(field=field):
|
|
other = self.node(2); other[field] = self.plan['nodes'][0][field]
|
|
p = copy.deepcopy(self.plan); p['nodes'].append(other)
|
|
with self.assertRaises(ValueError): batch.validate(p)
|
|
p = copy.deepcopy(self.plan); p['nodes'][0]['name'] = '🇯🇵 日本 JP - 11'
|
|
with self.assertRaises(ValueError): batch.validate(p)
|
|
|
|
def test_preflight_barrier_blocks_every_write(self):
|
|
self.plan['nodes'].append(self.node(2)); self.plan['block_preflight'] = True
|
|
self.run_cli(expected=2)
|
|
self.assertEqual(self.events(), [])
|
|
|
|
def test_parallel_nodes_serial_panel_and_resume(self):
|
|
self.plan['nodes'] = [self.node(i) for i in (1,2,3)]
|
|
self.run_cli()
|
|
events = self.events()
|
|
for phase, expected_peak in [('stage',1), ('publish',1), ('provision',None)]:
|
|
active = peak = 0
|
|
for e in sorted((e for e in events if e['phase']==phase), key=lambda e:e['t']):
|
|
active += 1 if e['event']=='start' else -1; peak=max(peak,active)
|
|
if expected_peak is None:
|
|
self.assertGreater(peak,1,'Independent nodes should overlap')
|
|
self.assertLessEqual(peak,3)
|
|
else:
|
|
self.assertEqual(peak, expected_peak, phase)
|
|
self.run_cli()
|
|
self.assertEqual(self.events(), events, 'A completed retry must not write again')
|
|
status = self.run_cli('status')
|
|
self.assertTrue(all(n['audit']=='done' for n in status.values()))
|
|
self.assertEqual(self.state.stat().st_mode & 0o777, 0o700)
|
|
self.assertEqual((self.state/'registry.sqlite3').stat().st_mode & 0o777, 0o600)
|
|
|
|
def test_lost_response_recovers_by_probe_without_duplicate(self):
|
|
self.plan['crash'] = True
|
|
self.run_cli(expected=2)
|
|
self.run_cli()
|
|
writes = [e for e in self.events() if e['phase']=='provision' and e['event']=='start']
|
|
self.assertEqual(len(writes),1)
|
|
self.assertEqual(self.run_cli('status')['jp1']['audit'],'done')
|
|
|
|
def test_cross_batch_and_plan_drift_rejected_before_commands(self):
|
|
self.run_cli(); events = self.events()
|
|
self.plan['nodes'][0]['expected_exit'] = '192.0.2.99'
|
|
self.run_cli(expected=2)
|
|
self.plan['batch_id'] = 'other-batch'
|
|
self.run_cli(expected=2)
|
|
self.assertEqual(self.events(), events)
|
|
|
|
def test_two_processes_share_locks_and_do_not_duplicate(self):
|
|
argv = self.argv()
|
|
a = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
b = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
for process in (a,b):
|
|
out, err = process.communicate(timeout=30)
|
|
self.assertEqual(process.returncode,0,out+err)
|
|
stages = [e for e in self.events() if e['phase']=='stage' and e['event']=='start']
|
|
self.assertEqual(len(stages),1)
|
|
|
|
def test_invalid_receipt_cannot_authorize_write(self):
|
|
self.plan['bad_receipt'] = True
|
|
self.run_cli(expected=2)
|
|
self.assertEqual(self.events(), [])
|
|
|
|
def test_interrupted_apply_records_unknown_and_can_resume(self):
|
|
p = subprocess.Popen(self.argv(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
try:
|
|
deadline = time.monotonic() + 10
|
|
while not any(e['phase']=='provision' and e['event']=='start' for e in self.events()):
|
|
self.assertIsNone(p.poll())
|
|
self.assertLess(time.monotonic(), deadline)
|
|
time.sleep(.02)
|
|
p.send_signal(signal.SIGTERM)
|
|
out, err = p.communicate(timeout=10)
|
|
self.assertEqual(p.returncode,2,out+err)
|
|
self.assertEqual(self.run_cli('status')['jp1']['provision'],'unknown')
|
|
self.run_cli()
|
|
self.assertEqual(self.run_cli('status')['jp1']['audit'],'done')
|
|
finally:
|
|
if p.poll() is None:
|
|
p.kill(); p.communicate()
|
|
|
|
def test_configured_command_adapter_runs_full_protocol(self):
|
|
self.plan['adapter'] = [sys.executable,str(ADAPTER)]
|
|
command = [sys.executable,str(self.fake),'--plan','{plan}','--node','{node}',
|
|
'--phase','{phase}','--action','{action}','--state-dir','{state_dir}']
|
|
self.plan['command_profile'] = {
|
|
'files': {str(self.fake): hashlib.sha256(self.fake.read_bytes()).hexdigest()},
|
|
'phases': {phase: {'probe': command, 'apply': command}
|
|
for phase in ('preflight',) + batch.PHASES}}
|
|
self.run_cli()
|
|
self.assertEqual(self.run_cli('status')['jp1']['audit'],'done')
|
|
|
|
def test_command_adapter_pins_handlers_and_missing_steps_wait(self):
|
|
self.plan['adapter'] = [sys.executable,str(ADAPTER)]
|
|
self.plan['command_profile'] = {'files': {str(self.fake): hashlib.sha256(self.fake.read_bytes()).hexdigest()},
|
|
'phases': {}}
|
|
self.run_cli(expected=2)
|
|
self.assertEqual(self.run_cli('status')['jp1']['preflight'],'waiting')
|
|
self.fake.write_text('# changed')
|
|
self.run_cli(expected=2)
|
|
self.assertEqual(self.run_cli('status')['jp1']['preflight'],'unknown')
|
|
|
|
|
|
if __name__ == '__main__': unittest.main()
|