102 lines
3.4 KiB
Python
102 lines
3.4 KiB
Python
#!/usr/bin/python3
|
|
"""Bound only Gitea's regenerable repository download archives, never repositories."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import stat
|
|
import time
|
|
|
|
ROOT = Path('/var/lib/gitea/data/repo-archive')
|
|
BUDGET = 512 * 1024 * 1024
|
|
TTL = 3600
|
|
GRACE = 600
|
|
ARCHIVE = re.compile(r'([0-9a-f]{40})\.(?:zip|tar\.gz|bundle)\Z')
|
|
|
|
|
|
def open_inodes():
|
|
result = set()
|
|
for proc in Path('/proc').iterdir():
|
|
if not proc.name.isdigit():
|
|
continue
|
|
try:
|
|
for fd in (proc / 'fd').iterdir():
|
|
try:
|
|
s = fd.stat()
|
|
if stat.S_ISREG(s.st_mode):
|
|
result.add((s.st_dev, s.st_ino))
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
pass
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
pass
|
|
return result
|
|
|
|
|
|
def clean(root, now, opened, dry_run=False, budget=BUDGET, ttl=TTL, grace=GRACE):
|
|
if root.is_symlink() or root.resolve() != root or not root.is_dir():
|
|
raise RuntimeError('cache_root_missing_or_symlink')
|
|
files = []
|
|
unknown = 0
|
|
for base, dirs, names in os.walk(root, followlinks=False):
|
|
dirs[:] = [d for d in dirs if not Path(base, d).is_symlink()]
|
|
for name in names:
|
|
p = Path(base, name)
|
|
try:
|
|
s = p.lstat()
|
|
except FileNotFoundError:
|
|
continue
|
|
if not stat.S_ISREG(s.st_mode):
|
|
continue
|
|
parts = p.relative_to(root).parts
|
|
match = ARCHIVE.fullmatch(name)
|
|
known = len(parts) == 3 and parts[0].isdigit() and match and parts[1] == match[1][:2]
|
|
if not known:
|
|
unknown += 1
|
|
files.append((p, s, bool(known)))
|
|
total = sum(s.st_size for _, s, _ in files)
|
|
before = total
|
|
removed = 0
|
|
protected = 0
|
|
for p, s, known in sorted(files, key=lambda row: row[1].st_mtime):
|
|
age = now - s.st_mtime
|
|
if not known or (s.st_dev, s.st_ino) in opened or age < grace:
|
|
protected += 1
|
|
continue
|
|
if age < ttl and total <= budget:
|
|
continue
|
|
try:
|
|
current = p.lstat()
|
|
except FileNotFoundError:
|
|
continue
|
|
if (current.st_dev, current.st_ino, current.st_size, current.st_mtime_ns) != (s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns):
|
|
continue
|
|
if not dry_run:
|
|
p.unlink()
|
|
total -= s.st_size
|
|
removed += 1
|
|
return dict(dry_run=dry_run, before_bytes=before, after_bytes=total,
|
|
removed_files=removed, protected_files=protected, unknown_files=unknown,
|
|
budget_bytes=budget, over_budget=total > budget)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--dry-run', action='store_true')
|
|
args = parser.parse_args()
|
|
with open('/run/gitea-archive-cache-cleaner/cleanup.lock', 'w') as lock:
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
return
|
|
result = clean(ROOT, time.time(), open_inodes(), args.dry_run)
|
|
result['free_bytes'] = os.statvfs(ROOT).f_bavail * os.statvfs(ROOT).f_frsize
|
|
print(json.dumps(result), flush=True)
|
|
if result['over_budget']:
|
|
raise SystemExit('Cache remains over budget: fresh/open/unknown files are protected')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|