204 行
8.6 KiB
Python
204 行
8.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Publish master from dedicated Git caches; retain production data/configuration."""
|
|
import argparse
|
|
import datetime
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import logging
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import shutil
|
|
import shlex
|
|
import subprocess
|
|
import tarfile
|
|
import tempfile
|
|
import urllib.request
|
|
|
|
LOG = logging.getLogger('grain-deploy')
|
|
|
|
|
|
def run(args):
|
|
result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=180)
|
|
if result.returncode:
|
|
raise RuntimeError('Command failed: {}: {}'.format(args[0], result.stderr.decode(errors='replace')[-1500:]))
|
|
return result.stdout
|
|
|
|
|
|
def selected(name, kind):
|
|
parts = PurePosixPath(name).parts
|
|
if not parts or any(p in ('.', '..') or p.startswith('.') for p in parts):
|
|
return False
|
|
if kind == 'frontend':
|
|
return name in ('index.html', 'xishui.html') or parts[0] in ('css', 'js', 'assets')
|
|
if name in ('application/database.php', 'application/config.php', 'application/extra/site.php'):
|
|
return False
|
|
if parts[0] == 'public' and len(parts) > 1 and parts[1] in ('uploads', 'screen', 'install'):
|
|
return False
|
|
return parts[0] in ('application', 'public', 'extend', 'addons')
|
|
|
|
|
|
def atomic_json(path, data):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temp = path.with_suffix('.tmp')
|
|
temp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
|
|
os.chmod(str(temp), 0o600)
|
|
os.replace(str(temp), str(path))
|
|
|
|
|
|
def destination(root, name):
|
|
result = root / name
|
|
if os.path.commonpath([str(root.resolve()), str(result.resolve())]) != str(root.resolve()):
|
|
raise RuntimeError('Deployment path escapes site root: ' + name)
|
|
return result
|
|
|
|
|
|
def healthcheck(url, host):
|
|
req = urllib.request.Request(url, headers={'Host': host, 'Cache-Control': 'no-cache'})
|
|
with urllib.request.urlopen(req, timeout=20) as response:
|
|
body = response.read()
|
|
if response.status != 200:
|
|
raise RuntimeError('Health check failed: ' + url)
|
|
if '/granaryList' in url:
|
|
data = json.loads(body.decode('utf-8'))
|
|
if data.get('code') != 1 or not isinstance(data.get('data', {}).get('list'), list):
|
|
raise RuntimeError('Granary API health check failed')
|
|
|
|
|
|
def replace_file(source, live):
|
|
live.parent.mkdir(parents=True, exist_ok=True)
|
|
temp = live.with_name(live.name + '.deploy-tmp')
|
|
shutil.copyfile(str(source), str(temp))
|
|
mode = live.stat().st_mode & 0o777 if live.exists() else 0o644
|
|
os.chmod(str(temp), mode)
|
|
# The service runs as www; replacing an older root-uploaded file must not
|
|
# attempt to chown it back to root or require elevated permissions.
|
|
os.replace(str(temp), str(live))
|
|
|
|
|
|
def publish(config, repo):
|
|
state_dir = Path(config['state'])
|
|
cache = state_dir / (repo['name'] + '.git')
|
|
cache.parent.mkdir(parents=True, exist_ok=True)
|
|
if not cache.exists():
|
|
run(['git', 'init', '--bare', str(cache)])
|
|
git = ['git', '--git-dir=' + str(cache)]
|
|
if repo.get('key'):
|
|
ssh = 'ssh -i {} -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile={}'.format(
|
|
shlex.quote(repo['key']), shlex.quote(repo['known_hosts']))
|
|
git += ['-c', 'core.sshCommand=' + ssh]
|
|
run(git + ['fetch', '--depth=1', repo['url'], '+refs/heads/master:refs/heads/deploy'])
|
|
revision = run(git + ['rev-parse', 'refs/heads/deploy']).decode().strip()
|
|
state_path = state_dir / (repo['name'] + '.json')
|
|
state = json.loads(state_path.read_text()) if state_path.exists() else {}
|
|
if state.get('revision') == revision:
|
|
LOG.info('%s unchanged %s', repo['name'], revision[:12])
|
|
return False
|
|
root = Path(repo['target'])
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
timestamp = datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%S%fZ')
|
|
backup = Path(config['backups']) / (timestamp + '-' + repo['name'] + '-' + revision[:12])
|
|
with tempfile.TemporaryDirectory(prefix='grain-deploy-', dir=str(state_dir)) as directory:
|
|
stage = Path(directory)
|
|
manifest = {}
|
|
top_level = run(git + ['ls-tree', '--name-only', revision]).decode().splitlines()
|
|
prefixes = [name for name in top_level if selected(name, repo['name']) or selected(name + '/file', repo['name'])]
|
|
if not prefixes:
|
|
raise RuntimeError('No deployable source directories')
|
|
archive = run(git + ['archive', revision] + prefixes)
|
|
with tarfile.open(fileobj=io.BytesIO(archive)) as tar:
|
|
for entry in tar:
|
|
if not selected(entry.name, repo['name']):
|
|
continue
|
|
if entry.isdir():
|
|
continue
|
|
if not entry.isfile():
|
|
raise RuntimeError('Only regular deployment files are supported: ' + entry.name)
|
|
data = tar.extractfile(entry).read()
|
|
path = stage / entry.name
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_bytes(data)
|
|
manifest[entry.name] = hashlib.sha256(data).hexdigest()
|
|
if not manifest:
|
|
raise RuntimeError('Refusing empty deployment')
|
|
changes = []
|
|
for name, digest in manifest.items():
|
|
live = destination(root, name)
|
|
if not live.is_file() or hashlib.sha256(live.read_bytes()).hexdigest() != digest:
|
|
changes.append(name)
|
|
deleted = sorted(set(state.get('files', [])) - set(manifest))
|
|
deleted = [name for name in deleted if selected(name, repo['name'])]
|
|
for name in changes:
|
|
path = stage / name
|
|
if path.suffix == '.php':
|
|
run([config['php'], '-l', str(path)])
|
|
elif path.suffix == '.js' and shutil.which('node'):
|
|
# Browser ES modules are validated by the frontend test/build workflow.
|
|
if not name.startswith('js/'):
|
|
run(['node', '--check', str(path)])
|
|
backup.mkdir(parents=True, mode=0o700)
|
|
records = []
|
|
for name in sorted(set(changes + deleted)):
|
|
live = destination(root, name)
|
|
record = {'path': name, 'existed': live.is_file()}
|
|
if live.exists() and not live.is_file():
|
|
raise RuntimeError('Expected a regular file: ' + name)
|
|
if live.is_file():
|
|
saved = backup / 'files' / name
|
|
saved.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(str(live), str(saved))
|
|
records.append(record)
|
|
atomic_json(backup / 'manifest.json', {'root': str(root), 'files': records, 'revision': revision})
|
|
try:
|
|
for name in changes:
|
|
live = destination(root, name)
|
|
replace_file(stage / name, live)
|
|
for name in deleted:
|
|
live = destination(root, name)
|
|
if live.is_file():
|
|
live.unlink()
|
|
for url in config.get('health', []):
|
|
healthcheck(url, config['host'])
|
|
except Exception:
|
|
for record in reversed(records):
|
|
live = destination(root, record['path'])
|
|
if record['existed']:
|
|
replace_file(backup / 'files' / record['path'], live)
|
|
elif live.is_file():
|
|
live.unlink()
|
|
LOG.exception('%s rolled back; backup=%s', repo['name'], backup)
|
|
raise
|
|
atomic_json(state_path, {'revision': revision, 'deployed_at': timestamp, 'files': list(manifest), 'backup': str(backup)})
|
|
LOG.info('%s deployed %s changed=%d removed=%d backup=%s', repo['name'], revision[:12], len(changes), len(deleted), backup)
|
|
return True
|
|
|
|
|
|
def main():
|
|
import fcntl
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--config', default='/etc/grain-auto-deploy.json')
|
|
args = parser.parse_args()
|
|
config = json.loads(Path(args.config).read_text())
|
|
state = Path(config['state'])
|
|
state.mkdir(parents=True, exist_ok=True)
|
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s',
|
|
handlers=[logging.StreamHandler(), logging.FileHandler(config['log'])])
|
|
with (state / 'deploy.lock').open('w') as lock:
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
return
|
|
failures = 0
|
|
for repo in config['repos']:
|
|
try:
|
|
publish(config, repo)
|
|
except Exception:
|
|
failures += 1
|
|
LOG.exception('%s deployment failed', repo['name'])
|
|
if failures:
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|