"""Offline process-level tests. No Laravel bootstrap, .env, SSH or network.""" import copy import hashlib import importlib.util import json import os from pathlib import Path import signal import subprocess import sys import tempfile import time import unittest ROOT = Path(__file__).resolve().parents[2] RUNNER = ROOT / 'scripts/node-onboarding/batch.py' ADAPTER = ROOT / 'scripts/node-onboarding/command-adapter.py' spec = importlib.util.spec_from_file_location('dpl_batch', RUNNER) batch = importlib.util.module_from_spec(spec) spec.loader.exec_module(batch) FAKE = r''' import argparse, json, os, sys, time from pathlib import Path p=argparse.ArgumentParser() for x in ('plan','node','phase','action','state-dir'): p.add_argument('--'+x,required=True) a=p.parse_args(); n=json.loads(Path(a.node).read_text()); plan=json.loads(Path(a.plan).read_text()) root=Path(plan['fixture_root']); root.mkdir(exist_ok=True) identity=dict(key=n['key'], phase=a.phase, request_sha256=n['request_sha256']) def emit(status, **kw): print(json.dumps(dict(identity,status=status,**kw))) if plan.get('bad_receipt'): identity['request_sha256']='wrong' if a.phase=='preflight': emit('waiting' if plan.get('block_preflight') and n['key']=='jp2' else 'ready'); sys.exit() if a.phase in ('verify','audit'): emit('ready'); sys.exit() marker=root/(n['key']+'-'+a.phase) if a.action=='probe': emit('ready' if marker.exists() else 'absent',retry_safe=True); sys.exit() def log(event): fd=os.open(root/'events',os.O_CREAT|os.O_WRONLY|os.O_APPEND,0o600) os.write(fd,(json.dumps(dict(key=n['key'],phase=a.phase,event=event,t=time.monotonic()))+'\n').encode());os.close(fd) log('start'); time.sleep(1.2 if a.phase=='provision' else .12); marker.write_text('owned'); log('end') if plan.get('crash') and a.phase=='provision' and not (root/'crashed').exists(): (root/'crashed').write_text('yes'); sys.exit(8) emit('ready') ''' class BatchTests(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.root = Path(self.tmp.name) self.fake = self.root / 'fake.py' self.fake.write_text(FAKE) self.state = self.root / 'state' self.plan = { 'version': 1, 'mode': 'append', 'batch_id': 'offline-test', 'context': {'panel_vps': 'VPS-001', 'gateway_vps': 'VPS-002', 'jump_vps': 'VPS-003', 'app_root': '/app', 'panel_url': 'https://panel.example.invalid', 'subscription_url': 'https://sub.example.invalid', 'provision_commit': 'a' * 40, 'bundle': '/private/bundle', 'client_binary': '/private/sing-box'}, 'adapter': [sys.executable, str(self.fake)], 'fixture_root': str(self.root / 'remote'), 'nodes': [self.node(1)], } self.path = self.root / 'plan.json' def tearDown(self): self.tmp.cleanup() def node(self, i): return dict(key=f'jp{i}', country='JP', sequence=i, flag='🇯🇵', country_name='日本', name=f'🇯🇵 日本 JP - {i:02d}', vps_id=f'VPS-{10+i:03d}', node_address=f'192.0.2.{i}', listen_port=24443, relay_host='relay.example.invalid', relay_port=47000+i, tls_domain=f'jp{i}.example.invalid', formal_group_id=1, expected_exit=f'192.0.2.{i}', socks_port=19000+i) def argv(self, command='run'): self.path.write_text(json.dumps(self.plan, ensure_ascii=False)) return [sys.executable, str(RUNNER), command, '--plan', str(self.path), '--state-root', str(self.state), '--jobs', '3'] + (['--execute'] if command == 'run' else []) def run_cli(self, command='run', expected=0): p = subprocess.run(self.argv(command), capture_output=True, text=True, timeout=30) self.assertEqual(p.returncode, expected, p.stdout + p.stderr) return json.loads(p.stdout) def events(self): path = self.root / 'remote/events' return [json.loads(x) for x in path.read_text().splitlines()] if path.exists() else [] def test_plan_and_status_are_read_only(self): self.run_cli('plan'); self.run_cli('status') self.assertFalse(self.state.exists()) self.assertFalse((self.root/'remote').exists()) def test_duplicate_parameters_and_naming_rejected(self): for field in ('key','name','vps_id','node_address','tls_domain','relay_port','socks_port','sequence'): with self.subTest(field=field): other = self.node(2); other[field] = self.plan['nodes'][0][field] p = copy.deepcopy(self.plan); p['nodes'].append(other) with self.assertRaises(ValueError): batch.validate(p) p = copy.deepcopy(self.plan); p['nodes'][0]['name'] = '🇯🇵 日本 JP - 11' with self.assertRaises(ValueError): batch.validate(p) def test_preflight_barrier_blocks_every_write(self): self.plan['nodes'].append(self.node(2)); self.plan['block_preflight'] = True self.run_cli(expected=2) self.assertEqual(self.events(), []) def test_parallel_nodes_serial_panel_and_resume(self): self.plan['nodes'] = [self.node(i) for i in (1,2,3)] self.run_cli() events = self.events() for phase, expected_peak in [('stage',1), ('publish',1), ('provision',None)]: active = peak = 0 for e in sorted((e for e in events if e['phase']==phase), key=lambda e:e['t']): active += 1 if e['event']=='start' else -1; peak=max(peak,active) if expected_peak is None: self.assertGreater(peak,1,'Independent nodes should overlap') self.assertLessEqual(peak,3) else: self.assertEqual(peak, expected_peak, phase) self.run_cli() self.assertEqual(self.events(), events, 'A completed retry must not write again') status = self.run_cli('status') self.assertTrue(all(n['audit']=='done' for n in status.values())) self.assertEqual(self.state.stat().st_mode & 0o777, 0o700) self.assertEqual((self.state/'registry.sqlite3').stat().st_mode & 0o777, 0o600) def test_lost_response_recovers_by_probe_without_duplicate(self): self.plan['crash'] = True self.run_cli(expected=2) self.run_cli() writes = [e for e in self.events() if e['phase']=='provision' and e['event']=='start'] self.assertEqual(len(writes),1) self.assertEqual(self.run_cli('status')['jp1']['audit'],'done') def test_cross_batch_and_plan_drift_rejected_before_commands(self): self.run_cli(); events = self.events() self.plan['nodes'][0]['expected_exit'] = '192.0.2.99' self.run_cli(expected=2) self.plan['batch_id'] = 'other-batch' self.run_cli(expected=2) self.assertEqual(self.events(), events) def test_two_processes_share_locks_and_do_not_duplicate(self): argv = self.argv() a = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) b = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) for process in (a,b): out, err = process.communicate(timeout=30) self.assertEqual(process.returncode,0,out+err) stages = [e for e in self.events() if e['phase']=='stage' and e['event']=='start'] self.assertEqual(len(stages),1) def test_invalid_receipt_cannot_authorize_write(self): self.plan['bad_receipt'] = True self.run_cli(expected=2) self.assertEqual(self.events(), []) def test_interrupted_apply_records_unknown_and_can_resume(self): p = subprocess.Popen(self.argv(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) try: deadline = time.monotonic() + 10 while not any(e['phase']=='provision' and e['event']=='start' for e in self.events()): self.assertIsNone(p.poll()) self.assertLess(time.monotonic(), deadline) time.sleep(.02) p.send_signal(signal.SIGTERM) out, err = p.communicate(timeout=10) self.assertEqual(p.returncode,2,out+err) self.assertEqual(self.run_cli('status')['jp1']['provision'],'unknown') self.run_cli() self.assertEqual(self.run_cli('status')['jp1']['audit'],'done') finally: if p.poll() is None: p.kill(); p.communicate() def test_configured_command_adapter_runs_full_protocol(self): self.plan['adapter'] = [sys.executable,str(ADAPTER)] command = [sys.executable,str(self.fake),'--plan','{plan}','--node','{node}', '--phase','{phase}','--action','{action}','--state-dir','{state_dir}'] self.plan['command_profile'] = { 'files': {str(self.fake): hashlib.sha256(self.fake.read_bytes()).hexdigest()}, 'phases': {phase: {'probe': command, 'apply': command} for phase in ('preflight',) + batch.PHASES}} self.run_cli() self.assertEqual(self.run_cli('status')['jp1']['audit'],'done') def test_command_adapter_pins_handlers_and_missing_steps_wait(self): self.plan['adapter'] = [sys.executable,str(ADAPTER)] self.plan['command_profile'] = {'files': {str(self.fake): hashlib.sha256(self.fake.read_bytes()).hexdigest()}, 'phases': {}} self.run_cli(expected=2) self.assertEqual(self.run_cli('status')['jp1']['preflight'],'waiting') self.fake.write_text('# changed') self.run_cli(expected=2) self.assertEqual(self.run_cli('status')['jp1']['preflight'],'unknown') if __name__ == '__main__': unittest.main()