#!/usr/bin/env python3 """Publish master from dedicated Git caches; retain production data/configuration.""" import argparse import datetime import hashlib import io import json import logging import os from pathlib import Path, PurePosixPath import shutil import subprocess import tarfile import tempfile import urllib.request LOG = logging.getLogger('grain-deploy') def run(args): result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=180) if result.returncode: raise RuntimeError('Command failed: {}: {}'.format(args[0], result.stderr.decode(errors='replace')[-1500:])) return result.stdout def selected(name, kind): parts = PurePosixPath(name).parts if not parts or any(p in ('.', '..') or p.startswith('.') for p in parts): return False if kind == 'frontend': return name in ('index.html', 'xishui.html') or parts[0] in ('css', 'js', 'assets') if name in ('application/database.php', 'application/config.php', 'application/extra/site.php'): return False if parts[0] == 'public' and len(parts) > 1 and parts[1] in ('uploads', 'screen', 'install'): return False return parts[0] in ('application', 'public', 'extend', 'addons') def atomic_json(path, data): path.parent.mkdir(parents=True, exist_ok=True) temp = path.with_suffix('.tmp') temp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8') os.chmod(str(temp), 0o600) os.replace(str(temp), str(path)) def destination(root, name): result = root / name if os.path.commonpath([str(root.resolve()), str(result.resolve())]) != str(root.resolve()): raise RuntimeError('Deployment path escapes site root: ' + name) return result def healthcheck(url, host): req = urllib.request.Request(url, headers={'Host': host, 'Cache-Control': 'no-cache'}) with urllib.request.urlopen(req, timeout=20) as response: body = response.read() if response.status != 200: raise RuntimeError('Health check failed: ' + url) if '/granaryList' in url: data = json.loads(body.decode('utf-8')) if data.get('code') != 1 or not isinstance(data.get('data', {}).get('list'), list): raise RuntimeError('Granary API health check failed') def publish(config, repo): state_dir = Path(config['state']) cache = state_dir / (repo['name'] + '.git') cache.parent.mkdir(parents=True, exist_ok=True) if not cache.exists(): run(['git', 'init', '--bare', str(cache)]) git = ['git', '--git-dir=' + str(cache)] run(git + ['fetch', '--depth=1', repo['url'], '+refs/heads/master:refs/heads/deploy']) revision = run(git + ['rev-parse', 'refs/heads/deploy']).decode().strip() state_path = state_dir / (repo['name'] + '.json') state = json.loads(state_path.read_text()) if state_path.exists() else {} if state.get('revision') == revision: LOG.info('%s unchanged %s', repo['name'], revision[:12]) return False root = Path(repo['target']) root.mkdir(parents=True, exist_ok=True) timestamp = datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%S%fZ') backup = Path(config['backups']) / (timestamp + '-' + repo['name'] + '-' + revision[:12]) with tempfile.TemporaryDirectory(prefix='grain-deploy-', dir=str(state_dir)) as directory: stage = Path(directory) manifest = {} top_level = run(git + ['ls-tree', '--name-only', revision]).decode().splitlines() prefixes = [name for name in top_level if selected(name, repo['name']) or selected(name + '/file', repo['name'])] if not prefixes: raise RuntimeError('No deployable source directories') archive = run(git + ['archive', revision] + prefixes) with tarfile.open(fileobj=io.BytesIO(archive)) as tar: for entry in tar: if not selected(entry.name, repo['name']): continue if entry.isdir(): continue if not entry.isfile(): raise RuntimeError('Only regular deployment files are supported: ' + entry.name) data = tar.extractfile(entry).read() path = stage / entry.name path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(data) manifest[entry.name] = hashlib.sha256(data).hexdigest() if not manifest: raise RuntimeError('Refusing empty deployment') changes = [] for name, digest in manifest.items(): live = destination(root, name) if not live.is_file() or hashlib.sha256(live.read_bytes()).hexdigest() != digest: changes.append(name) deleted = sorted(set(state.get('files', [])) - set(manifest)) deleted = [name for name in deleted if selected(name, repo['name'])] for name in changes: path = stage / name if path.suffix == '.php': run([config['php'], '-l', str(path)]) elif path.suffix == '.js' and shutil.which('node'): # Browser ES modules are validated by the frontend test/build workflow. if not name.startswith('js/'): run(['node', '--check', str(path)]) backup.mkdir(parents=True, mode=0o700) records = [] for name in sorted(set(changes + deleted)): live = destination(root, name) record = {'path': name, 'existed': live.is_file()} if live.exists() and not live.is_file(): raise RuntimeError('Expected a regular file: ' + name) if live.is_file(): saved = backup / 'files' / name saved.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(str(live), str(saved)) records.append(record) atomic_json(backup / 'manifest.json', {'root': str(root), 'files': records, 'revision': revision}) try: for name in changes: live = destination(root, name) live.parent.mkdir(parents=True, exist_ok=True) temp = live.with_name(live.name + '.deploy-tmp') shutil.copyfile(str(stage / name), str(temp)) if live.exists(): stat = live.stat() os.chmod(str(temp), stat.st_mode & 0o777) if hasattr(os, 'chown'): os.chown(str(temp), stat.st_uid, stat.st_gid) else: os.chmod(str(temp), 0o644) if hasattr(os, 'chown'): parent = live.parent.stat() os.chown(str(temp), parent.st_uid, parent.st_gid) os.replace(str(temp), str(live)) for name in deleted: live = destination(root, name) if live.is_file(): live.unlink() for url in config.get('health', []): healthcheck(url, config['host']) except Exception: for record in reversed(records): live = destination(root, record['path']) if record['existed']: shutil.copy2(str(backup / 'files' / record['path']), str(live)) elif live.is_file(): live.unlink() LOG.exception('%s rolled back; backup=%s', repo['name'], backup) raise atomic_json(state_path, {'revision': revision, 'deployed_at': timestamp, 'files': list(manifest), 'backup': str(backup)}) LOG.info('%s deployed %s changed=%d removed=%d backup=%s', repo['name'], revision[:12], len(changes), len(deleted), backup) return True def main(): import fcntl parser = argparse.ArgumentParser() parser.add_argument('--config', default='/etc/grain-auto-deploy.json') args = parser.parse_args() config = json.loads(Path(args.config).read_text()) state = Path(config['state']) state.mkdir(parents=True, exist_ok=True) logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s', handlers=[logging.StreamHandler(), logging.FileHandler(config['log'])]) with (state / 'deploy.lock').open('w') as lock: try: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: return failures = 0 for repo in config['repos']: try: publish(config, repo) except Exception: failures += 1 LOG.exception('%s deployment failed', repo['name']) if failures: raise SystemExit(1) if __name__ == '__main__': main()