#!/usr/bin/env python3 """Publish master from dedicated Git caches; retain production data/configuration.""" import argparse import datetime import hashlib import io import json import logging import os from pathlib import Path, PurePosixPath import shutil import shlex import subprocess import tarfile import tempfile import urllib.request LOG = logging.getLogger('grain-deploy') def run(args): result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=180) if result.returncode: raise RuntimeError('Command failed: {}: {}'.format(args[0], result.stderr.decode(errors='replace')[-1500:])) return result.stdout def selected(name, kind): parts = PurePosixPath(name).parts if not parts or any(p in ('.', '..') or p.startswith('.') for p in parts): return False if kind == 'frontend': return name in ('index.html', 'xishui.html') or parts[0] in ('css', 'js', 'assets') if name in ('application/database.php', 'application/config.php', 'application/extra/site.php'): return False if parts[0] == 'public' and len(parts) > 1 and parts[1] in ('uploads', 'screen', 'install'): return False return parts[0] in ('application', 'public', 'extend', 'addons') def atomic_json(path, data): path.parent.mkdir(parents=True, exist_ok=True) temp = path.with_suffix('.tmp') temp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8') os.chmod(str(temp), 0o600) os.replace(str(temp), str(path)) def destination(root, name): result = root / name if os.path.commonpath([str(root.resolve()), str(result.resolve())]) != str(root.resolve()): raise RuntimeError('Deployment path escapes site root: ' + name) return result def healthcheck(url, host): req = urllib.request.Request(url, headers={'Host': host, 'Cache-Control': 'no-cache'}) with urllib.request.urlopen(req, timeout=20) as response: body = response.read() if response.status != 200: raise RuntimeError('Health check failed: ' + url) if '/granaryList' in url: data = json.loads(body.decode('utf-8')) if data.get('code') != 1 or not isinstance(data.get('data', {}).get('list'), list): raise RuntimeError('Granary API health check failed') def replace_file(source, live): live.parent.mkdir(parents=True, exist_ok=True) temp = live.with_name(live.name + '.deploy-tmp') shutil.copyfile(str(source), str(temp)) mode = live.stat().st_mode & 0o777 if live.exists() else 0o644 os.chmod(str(temp), mode) # The service runs as www; replacing an older root-uploaded file must not # attempt to chown it back to root or require elevated permissions. os.replace(str(temp), str(live)) def publish(config, repo): state_dir = Path(config['state']) cache = state_dir / (repo['name'] + '.git') cache.parent.mkdir(parents=True, exist_ok=True) if not cache.exists(): run(['git', 'init', '--bare', str(cache)]) git = ['git', '--git-dir=' + str(cache)] if repo.get('key'): ssh = 'ssh -i {} -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile={}'.format( shlex.quote(repo['key']), shlex.quote(repo['known_hosts'])) git += ['-c', 'core.sshCommand=' + ssh] run(git + ['fetch', '--depth=1', repo['url'], '+refs/heads/master:refs/heads/deploy']) revision = run(git + ['rev-parse', 'refs/heads/deploy']).decode().strip() state_path = state_dir / (repo['name'] + '.json') state = json.loads(state_path.read_text()) if state_path.exists() else {} if state.get('revision') == revision: LOG.info('%s unchanged %s', repo['name'], revision[:12]) return False root = Path(repo['target']) root.mkdir(parents=True, exist_ok=True) timestamp = datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%S%fZ') backup = Path(config['backups']) / (timestamp + '-' + repo['name'] + '-' + revision[:12]) with tempfile.TemporaryDirectory(prefix='grain-deploy-', dir=str(state_dir)) as directory: stage = Path(directory) manifest = {} top_level = run(git + ['ls-tree', '--name-only', revision]).decode().splitlines() prefixes = [name for name in top_level if selected(name, repo['name']) or selected(name + '/file', repo['name'])] if not prefixes: raise RuntimeError('No deployable source directories') archive = run(git + ['archive', revision] + prefixes) with tarfile.open(fileobj=io.BytesIO(archive)) as tar: for entry in tar: if not selected(entry.name, repo['name']): continue if entry.isdir(): continue if not entry.isfile(): raise RuntimeError('Only regular deployment files are supported: ' + entry.name) data = tar.extractfile(entry).read() path = stage / entry.name path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(data) manifest[entry.name] = hashlib.sha256(data).hexdigest() if not manifest: raise RuntimeError('Refusing empty deployment') changes = [] for name, digest in manifest.items(): live = destination(root, name) if not live.is_file() or hashlib.sha256(live.read_bytes()).hexdigest() != digest: changes.append(name) deleted = sorted(set(state.get('files', [])) - set(manifest)) deleted = [name for name in deleted if selected(name, repo['name'])] for name in changes: path = stage / name if path.suffix == '.php': run([config['php'], '-l', str(path)]) elif path.suffix == '.js' and shutil.which('node'): # Browser ES modules are validated by the frontend test/build workflow. if not name.startswith('js/'): run(['node', '--check', str(path)]) backup.mkdir(parents=True, mode=0o700) records = [] for name in sorted(set(changes + deleted)): live = destination(root, name) record = {'path': name, 'existed': live.is_file()} if live.exists() and not live.is_file(): raise RuntimeError('Expected a regular file: ' + name) if live.is_file(): saved = backup / 'files' / name saved.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(str(live), str(saved)) records.append(record) atomic_json(backup / 'manifest.json', {'root': str(root), 'files': records, 'revision': revision}) try: for name in changes: live = destination(root, name) replace_file(stage / name, live) for name in deleted: live = destination(root, name) if live.is_file(): live.unlink() for url in config.get('health', []): healthcheck(url, config['host']) except Exception: for record in reversed(records): live = destination(root, record['path']) if record['existed']: replace_file(backup / 'files' / record['path'], live) elif live.is_file(): live.unlink() LOG.exception('%s rolled back; backup=%s', repo['name'], backup) raise atomic_json(state_path, {'revision': revision, 'deployed_at': timestamp, 'files': list(manifest), 'backup': str(backup)}) LOG.info('%s deployed %s changed=%d removed=%d backup=%s', repo['name'], revision[:12], len(changes), len(deleted), backup) return True def main(): import fcntl parser = argparse.ArgumentParser() parser.add_argument('--config', default='/etc/grain-auto-deploy.json') args = parser.parse_args() config = json.loads(Path(args.config).read_text()) state = Path(config['state']) state.mkdir(parents=True, exist_ok=True) logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s', handlers=[logging.StreamHandler(), logging.FileHandler(config['log'])]) with (state / 'deploy.lock').open('w') as lock: try: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: return failures = 0 for repo in config['repos']: try: publish(config, repo) except Exception: failures += 1 LOG.exception('%s deployment failed', repo['name']) if failures: raise SystemExit(1) if __name__ == '__main__': main()