"""Online SQLite snapshots with optional copy to a pre-mounted network directory.""" import datetime as dt import calendar from contextlib import closing import hashlib import os from pathlib import Path, PurePosixPath import re import secrets import shutil import sqlite3 import threading import time class Backups: def __init__(self, app): self.a = app self.guard = threading.Lock() self.running = False self.stage = '' def config(self): return {'baidu': False, **self.a.setting('backup_config', dict(enabled=False, time='03:00', keep=14, networkPath='', revision=0))} def status(self): return dict(config=self.config(), running=self.running, stage=self.stage, localPath=str(self.a.DATA/'managed-backups'), last=self.a.setting('backup_last', None), history=self.a.setting('backup_history', [])[-20:][::-1]) @staticmethod def network_mount(folder): if os.name == 'nt': return folder.drive.startswith('\\\\') try: records = Path('/proc/self/mountinfo').read_text().splitlines() except OSError: return False return Backups.linux_network_mount(PurePosixPath(str(folder)), records) @staticmethod def linux_network_mount(folder, records): matches = [] for line in records: left, separator, right = line.partition(' - ') fields, detail = left.split(), right.split() if not separator or len(fields) < 5 or not detail: continue mount = PurePosixPath(re.sub(r'\\([0-7]{3})', lambda m: chr(int(m[1], 8)), fields[4])) if folder.is_relative_to(mount): matches.append((len(mount.parts), detail[0])) # The nearest mount wins: a tmpfs inside an NFS share is still local. filesystem = max(matches, default=(0, ''), key=lambda x:x[0])[1] return filesystem in {'cifs', 'smb3', 'nfs', 'nfs4', 'davfs', 'fuse.sshfs', 'fuse.rclone', 'fuse.davfs', 'fuse.davfs2', 'fuse.wdfs', 'fuse.s3fs', 'fuse.goofys'} def network_folder(self, text): if not text: return None if not isinstance(text, str) or len(text) > 500 or any(ord(c) < 32 for c in text) or '://' in text: raise self.a.Problem('请填写服务器已挂载的绝对目录,不能填写网页或分享链接') folder = Path(text) if not folder.is_absolute(): raise self.a.Problem('备份路径须为服务器上的绝对目录') folder = folder.resolve() if not folder.is_dir(): raise self.a.Problem('备份目录不存在,请先在服务器挂载网络存储并授予服务账号写入权限') # A disconnected mount must not silently become a local-only cloud backup. if not self.network_mount(folder): raise self.a.Problem('该目录未检测到支持的网络挂载,请先连接 NFS、SMB、WebDAV 或 rclone 网络盘') if folder == (self.a.DATA/'managed-backups').resolve(): raise self.a.Problem('网络副本与本机备份目录不能相同') return folder def save(self, data): data = {'baidu': False, **data} if set(data) != {'enabled','time','keep','networkPath','revision','baidu'}: raise self.a.Problem('备份设置字段不完整') with self.guard: if self.running: raise self.a.Problem('备份正在执行,请完成后再调整设置', 409) if not isinstance(data['baidu'], bool): raise self.a.Problem('网盘备份设置不正确') if data['baidu']: self.a.BAIDU.token('shared') with self.a.LOCK: old = self.config() if type(data['revision']) is not int or data['revision'] != old['revision']: raise self.a.Problem('设置已变化,请重新读取后保存', 409) if not isinstance(data['enabled'], bool) or not isinstance(data['time'], str) or not re.fullmatch(r'(?:[01]\d|2[0-3]):[0-5]\d', data['time']): raise self.a.Problem('请选择有效的每日备份时间') self.a.integer(data['keep'], 1, 90) if not isinstance(data['networkPath'], str): raise self.a.Problem('备份目录格式不正确') self.network_folder(data['networkPath']) self.a.set_setting('backup_config', dict(data, revision=old['revision']+1)) return self.status() def start(self, reason='manual'): with self.guard: if self.running: raise self.a.Problem('已有备份正在执行', 409) self.running = True try: threading.Thread(target=self.run, args=(reason,), daemon=True).start() except Exception: with self.guard: self.running = False raise return dict(started=True) def prune(self, folder, prefix, keep): owned = [p for p in folder.glob(prefix+'*.sqlite') if p.is_file() and not p.is_symlink()] for path in sorted(owned, key=lambda p:p.name, reverse=True)[keep:]: path.unlink() @staticmethod def month_before(now): year, month = (now.year, now.month-1) if now.month > 1 else (now.year-1, 12) return now.replace(year=year, month=month, day=min(now.day, calendar.monthrange(year, month)[1])) def prune_cloud(self, instance, newest, result, now=None): now = now or dt.datetime.now(dt.timezone(dt.timedelta(hours=8))) cutoff = self.month_before(now) result.update(cloudRetentionMonths=1, cloudDeleted=0, cloudCutoff=cutoff.isoformat()) root = '/apps/'+self.a.BAIDU.public_config()['appFolder']+'/数据库备份/'+instance pattern = re.compile('vision-'+re.escape(instance)+r'-(\d{8}T\d{12}Z)\.sqlite') # Collect all pages before deleting: changing a directory while paging can skip entries. for row in self.a.BAIDU.list_directory(root): path = row.get('path', '') if not path.startswith(root+'/'): continue name = path[len(root)+1:]; match = pattern.fullmatch(name) if not match: continue try: created = dt.datetime.strptime(match[1], '%Y%m%dT%H%M%S%fZ').replace(tzinfo=dt.timezone.utc) except ValueError: continue if created >= cutoff: continue candidates = self.a.BAIDU.list_directory(path) if row.get('isdir') == 1 else [row] for item in candidates: target = item.get('path', '') expected = path+'/'+name if row.get('isdir') == 1 else path if target != expected or target == newest or item.get('isdir') != 0: continue self.a.BAIDU.delete_backup_file(target) result['cloudDeleted'] += 1 result['cloudCleanupOk'] = True def run(self, reason='manual'): a = self.a result = dict(at=dt.datetime.now(dt.timezone.utc).isoformat(), reason=reason, localOk=False, networkOk=False, baiduOk=False, networkRequested=False, baiduRequested=False) temporary = network_temp = None try: config = self.config() result.update(networkRequested=bool(config['networkPath']), baiduRequested=config['baidu']) self.stage = '正在生成本机数据库副本' local = a.DATA/'managed-backups' local.mkdir(mode=0o700, exist_ok=True) instance = a.setting('backup_instance') if not instance: instance = secrets.token_hex(8); a.set_setting('backup_instance', instance) prefix = 'vision-'+instance+'-' name = prefix+dt.datetime.now(dt.timezone.utc).strftime('%Y%m%dT%H%M%S%fZ')+'.sqlite' target = local/name temporary = local/(name+'.partial') # Separate read connection avoids holding the live write lock during IO. with closing(sqlite3.connect(a.DATA/'vision.db', timeout=10)) as source, closing(sqlite3.connect(temporary)) as dest: source.backup(dest, pages=256, sleep=0.01) if dest.execute('PRAGMA quick_check').fetchone()[0] != 'ok': raise RuntimeError('数据库副本完整性检查失败') dest.execute('DELETE FROM sessions'); dest.commit() os.chmod(temporary, 0o600) with temporary.open('rb') as f: digest = hashlib.file_digest(f, 'sha256').hexdigest() os.replace(temporary, target) result.update(localOk=True, file=name, bytes=target.stat().st_size, sha256=digest) self.prune(local, prefix, config['keep']) if config['networkPath']: self.stage = '正在复制到网络目录' network = self.network_folder(config['networkPath']) network_temp = network/(name+'.partial') shutil.copyfile(target, network_temp) os.chmod(network_temp, 0o600) with network_temp.open('rb') as f: if hashlib.file_digest(f, 'sha256').hexdigest() != digest: raise RuntimeError('网络副本校验不一致') os.replace(network_temp, network/name) result['networkOk'] = True self.prune(network, prefix, config['keep']) if config['baidu']: self.stage = '正在上传百度网盘' remote = '/apps/'+a.BAIDU.public_config()['appFolder']+'/数据库备份/'+instance+'/'+name uploaded = a.BAIDU.upload('shared', target, remote) self.stage = '正在校验百度网盘副本' a.BAIDU.verify_backup(remote, uploaded['fs_id'], result['bytes'], digest) result.update(baiduOk=True, baiduPath=remote, baiduFileId=str(uploaded['fs_id'])) self.stage = '正在清理超过一个月的云端备份' try: self.prune_cloud(instance, remote, result) except Exception as error: result.update(cloudCleanupOk=False, cloudCleanupError=str(error)[:300] or '过期备份尚未清理') except Exception as error: result['error'] = str(error)[:300] or '备份未完成' finally: for temporary_path in (temporary, network_temp): try: if temporary_path and temporary_path.exists(): temporary_path.unlink() except OSError: pass try: with a.LOCK: a.set_setting('backup_last', result) history = a.setting('backup_history', []) a.set_setting('backup_history', (history+[result])[-100:]) finally: with self.guard: self.running = False; self.stage = '' return result def tick(self, now=None): now = now or dt.datetime.now(dt.timezone(dt.timedelta(hours=8))) config = self.config(); day = now.strftime('%Y-%m-%d') if not config['enabled'] or now.strftime('%H:%M') < config['time'] or self.running: return False if self.a.setting('backup_attempt_day') == day: return False self.a.set_setting('backup_attempt_day', day) self.start('scheduled') return True def watch(self): while True: try: self.tick() except Exception: pass time.sleep(60)