mirror of https://github.com/bvn13/PyBackuper.git
316 lines
11 KiB
Python
316 lines
11 KiB
Python
import os
|
|
import sys
|
|
import argparse
|
|
import glob
|
|
import ntpath
|
|
import traceback
|
|
from zipfile import ZipFile as ZipFile
|
|
|
|
import datetime
|
|
|
|
from pprint import pprint
|
|
|
|
from backuper.storer import BackupStorerYandexDisk
|
|
from backuper.sqlmanager import SQLManager
|
|
from backuper.dirwatcher import DirWatcher
|
|
from backuper.reporter import Reporter
|
|
|
|
from sets import settings
|
|
|
|
import threading
|
|
|
|
import logging
|
|
|
|
|
|
logging.basicConfig(level = logging.INFO, format='%(asctime)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S', filename=settings['log'])
|
|
|
|
def exception_hook(exc_type, exc_value, exc_traceback):
|
|
logging.error(
|
|
"Uncaught exception",
|
|
exc_info=(exc_type, exc_value, exc_traceback)
|
|
)
|
|
|
|
sys.excepthook = exception_hook
|
|
|
|
#raise Exception('Boom')
|
|
|
|
version = '0.3.0'
|
|
|
|
|
|
class Backuper(object) :
|
|
|
|
storer = None
|
|
sqlManager = None
|
|
|
|
localPath = ''
|
|
fileMask = '*'
|
|
makeZip = 'no'
|
|
|
|
databases = []
|
|
|
|
def init(self, settings) :
|
|
|
|
self.fileMask = settings['file_mask']
|
|
self.localPath = settings['local_path']
|
|
self.makeZip = settings['zip']['make']
|
|
|
|
storerType = settings['storer']
|
|
storerSettings = settings['storers'][storerType]
|
|
|
|
if (storerType == 'ya_disk') :
|
|
self.storer = BackupStorerYandexDisk()
|
|
self.storer.setPath(storerSettings['path'])
|
|
self.storer.setDomain(storerSettings['account']['domain'])
|
|
self.storer.setUsername(storerSettings['account']['username'])
|
|
self.storer.setPassword(storerSettings['account']['password'])
|
|
self.storer.setProtocol(storerSettings['account']['protocol'])
|
|
else :
|
|
raise NotImplementedError("%s storer is not implemented yet" % (storerType,))
|
|
|
|
sqlSettings = settings['sql']
|
|
|
|
self.sqlManager = SQLManager()
|
|
self.sqlManager.setServer(sqlSettings['server'])
|
|
self.sqlManager.setUsername(sqlSettings['username'])
|
|
self.sqlManager.setPassword(sqlSettings['password'])
|
|
|
|
self.databases = sqlSettings['databases']
|
|
return
|
|
|
|
|
|
def getLocalFiles(self, dirname, dbname="") :
|
|
print("file search mask: %s%s/%s/%s" % (self.localPath, dirname, dbname, self.fileMask))
|
|
return (glob.glob("%s%s/%s/%s" % (self.localPath, dirname, dbname, self.fileMask)))
|
|
|
|
|
|
def deleteOldLocalBackups(self) :
|
|
logging.info("deleting old backups")
|
|
for fname in self.getLocalFiles() : #glob.glob("%s%s" % (self.localPath, self.fileMask)) :
|
|
name = fname
|
|
if (self.makeZip == 'yes') :
|
|
name = "%s.zip" % (name,)
|
|
os.remove(name)
|
|
logging.info(" - deleted: %s" % (name,))
|
|
|
|
def zipFiles(files) :
|
|
newFiles = []
|
|
for fname in files :
|
|
zipFname = "%s.zip" % (fname,)
|
|
logging.info("creating zip: %s" % (zipFname,))
|
|
with ZipFile(zipFname, mode="w") as fzip :
|
|
logging.info(" - adding file: %s" % (fname,))
|
|
fzip.write(fname)
|
|
fzip.close()
|
|
newFiles.append(zipFname)
|
|
logging.info("deleting source file: %s" % (fname,))
|
|
os.remove(fname)
|
|
return newFiles
|
|
|
|
|
|
def backup(self, type='F', database=None, copyOnly=False) :
|
|
|
|
dirName = ''
|
|
if (type.upper() == 'F') :
|
|
dirName = 'full'
|
|
elif (type.upper() == 'D') :
|
|
dirName = 'diff'
|
|
elif (type.upper() == 'L') :
|
|
dirName = 'logs'
|
|
else :
|
|
raise NotImplementedError("Unknown type of database backups has been specified")
|
|
|
|
#self.deleteOldLocalBackups()
|
|
|
|
now = datetime.datetime.now()
|
|
nowDate = now.strftime("%Y%m%d")
|
|
nowTime = now.strftime("%H%M%S")
|
|
logging.info("now: %s %s" % (nowDate, nowTime))
|
|
|
|
databases = []
|
|
|
|
if not database :
|
|
databases = self.databases
|
|
else :
|
|
databases.append(database)
|
|
|
|
for dbname in databases :
|
|
if (not copyOnly) :
|
|
self.sqlManager.makeBackup(dbname, type, self.localPath)
|
|
|
|
locFiles = self.getLocalFiles(dirName, dbname)
|
|
if len(locFiles) == 0 :
|
|
logging.info("ERROR: could not find expected backup of database: %s" % (dbname,))
|
|
continue
|
|
if (self.makeZip == 'yes') :
|
|
locFiles = zipFiles(locFiles)
|
|
|
|
for file in locFiles :
|
|
self.storer.store(file, nowDate, nowTime, dirName)
|
|
|
|
#self.storer.manageStorage(setStorerSettings['left_days'])
|
|
|
|
|
|
|
|
class Watcher(object) :
|
|
|
|
storer = None
|
|
watcher = None
|
|
reporter = None
|
|
|
|
watchPath = ''
|
|
dirFullBackup = ''
|
|
dirDiffBackup = ''
|
|
fileMask = '*'
|
|
makeZip = 'no'
|
|
|
|
databases = []
|
|
files = set()
|
|
|
|
def init(self, settings) :
|
|
|
|
#self.fileMask = settings['file_mask']
|
|
|
|
self.watchPath = settings['watcher']['path']
|
|
self.dirFullBackup = settings['watcher']['dir_full']
|
|
self.dirDiffBackup = settings['watcher']['dir_diff'] if 'dir_diff' in settings['watcher'] else ''
|
|
self.dirLogsBackup = settings['watcher']['dir_logs'] if 'dir_logs' in settings['watcher'] else ''
|
|
|
|
self.makeZip = settings['zip']['make']
|
|
|
|
storerType = settings['storer']
|
|
storerSettings = settings['storers'][storerType]
|
|
|
|
self.reporter = Reporter()
|
|
self.reporter.init(settings)
|
|
|
|
if (storerType == 'ya_disk') :
|
|
self.storer = BackupStorerYandexDisk()
|
|
self.storer.setPath(storerSettings['path'])
|
|
self.storer.setDomain(storerSettings['account']['domain'])
|
|
self.storer.setUsername(storerSettings['account']['username'])
|
|
self.storer.setPassword(storerSettings['account']['password'])
|
|
self.storer.setProtocol(storerSettings['account']['protocol'])
|
|
self.storer.setDaysLeft(storerSettings['left_days'])
|
|
self.storer.setManageEnabled(storerSettings['manage_enabled'] == 'true')
|
|
else :
|
|
raise NotImplementedError("%s storer is not implemented yet" % (storerType,))
|
|
|
|
return
|
|
|
|
def _getCurrentTime(self) :
|
|
now = datetime.datetime.now()
|
|
nowDate = now.strftime("%Y%m%d")
|
|
nowTime = now.strftime("%H")
|
|
minutes = int(now.strftime("%M"))
|
|
if (minutes <= 15) :
|
|
nowTime += "00"
|
|
elif (minutes <= 30) :
|
|
nowTime += "15"
|
|
elif (minutes <= 45) :
|
|
nowTime += "30"
|
|
else :
|
|
nowTime += "45"
|
|
|
|
logging.info("now: %s %s" % (nowDate, nowTime))
|
|
return (nowDate, nowTime)
|
|
|
|
def onBackupCreated(self, filename) :
|
|
|
|
self.storer.manageStorage()
|
|
|
|
if not filename in self.files :
|
|
self.files.add(filename)
|
|
logging.info("BACKUP: %s" % (filename,))
|
|
full_path = "%s%s" % (self.watchPath, self.dirFullBackup)
|
|
logging.info("FULL PATH: %s" % (full_path,))
|
|
diff_path = "%s%s" % (self.watchPath, self.dirDiffBackup)
|
|
logging.info("DIFF PATH: %s" % (diff_path,))
|
|
logs_path = "%s%s" % (self.watchPath, self.dirLogsBackup)
|
|
logging.info("LOGS PATH: %s" % (logs_path,))
|
|
|
|
if (filename.find(full_path) >= 0) :
|
|
logging.info("FULL BACKUP")
|
|
|
|
t1 = threading.Timer(0, self._makeBackup, [filename, "FULL"])
|
|
t1.start()
|
|
|
|
elif (self.dirDiffBackup and filename.find(diff_path) >= 0) :
|
|
logging.info("DIFF BACKUP")
|
|
|
|
t2 = threading.Timer(0, self._makeBackup, [filename, "DIFF"])
|
|
t2.start()
|
|
|
|
elif (self.dirLogsBackup and filename.find(logs_path) >= 0) :
|
|
logging.info("LOGS BACKUP")
|
|
|
|
t2 = threading.Timer(0, self._makeBackup, [filename, "LOGS"])
|
|
t2.start()
|
|
|
|
else :
|
|
logging.warning("IGNORING: %s" % (filename,))
|
|
else :
|
|
logging.warning("SKIP DOUBLE TRIGGER FOR: %s" % (filename,))
|
|
|
|
return
|
|
|
|
def _makeBackup(self, filename, subfolder) :
|
|
(nowDate, nowTime) = self._getCurrentTime()
|
|
#fname = ntpath.basename(filename)
|
|
errortext = ""
|
|
remoteFname = ""
|
|
try :
|
|
remoteFname = self.storer.store(filename, nowDate, nowTime, subfolder)
|
|
self.reporter.report("%s BACKUP SUCCESSFULL" % (subfolder,), "%s BACKUP UPLOADED: %s -> %s" % (subfolder, filename, remoteFname))
|
|
logging.info("%s BACKUP UPLOADED: %s -> %s" % (subfolder, filename, remoteFname))
|
|
except :
|
|
trace = traceback.format_exc()
|
|
self.reporter.report("%s BACKUP ERROR" % (subfolder,), "%s BACKUP UPLOAD ERROR: %s - %s" % (subfolder, filename, trace))
|
|
logging.error("%s BACKUP UPLOAD ERROR: %s -> %s - %s" % (subfolder, filename, remoteFname, trace))
|
|
|
|
self.files.remove(filename)
|
|
return
|
|
|
|
def start(self) :
|
|
self.watcher = DirWatcher(onFileCreatedHandler=self.onBackupCreated, onFileModifiedHandler=self.onBackupCreated)
|
|
logging.info("Start watching files: %s%s" % (self.watchPath, self.fileMask))
|
|
self.watcher.watch(self.watchPath, self.fileMask)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
verstr = '%%(prog) %s' % (version,)
|
|
description = 'Backup maker. %s' % (verstr,)
|
|
|
|
parser = argparse.ArgumentParser(add_help=True, description='Backup Maker', prog='backuper')
|
|
parser.add_argument('-v', '--ver', '--version', action='version', version=version)
|
|
subparsers = parser.add_subparsers()
|
|
|
|
parser_watcher = subparsers.add_parser('watcher')
|
|
parser_watcher.add_argument('-s', '--start', required=True, dest='watcher', action='store_true')
|
|
|
|
parser_work = subparsers.add_parser('worker')
|
|
parser_work.add_argument('-t', '--type', metavar='type', required=True, type=str, choices=['F', 'D', 'L'], default='F', help='type of database backup to make this time: F - full, D - diff, L - logs')
|
|
parser_work.add_argument('-db', '--database', metavar='database', type=str, help='database name to backup. Will make backups for all databases (from settings) if not specified.')
|
|
parser_work.add_argument('-c', '--copy-only', action='store_true', dest='copy', help='Only copy files to storage')
|
|
|
|
args = parser.parse_args()
|
|
|
|
#pprint(args)
|
|
|
|
if not 'watcher' in args and not 'type' in args :
|
|
print("WRONG MODE: choose watcher/worker")
|
|
else :
|
|
if 'watcher' in args and args.watcher :
|
|
print("WATCH MODE")
|
|
watcher = Watcher()
|
|
watcher.init(settings)
|
|
watcher.start()
|
|
|
|
else :
|
|
backuper = Backuper()
|
|
backuper.init(settings)
|
|
if 'database' in args :
|
|
backuper.backup(type=args.type, database=args.database, copyOnly='copy' in args)
|
|
else :
|
|
backuper.backup(type=args.type, copyOnly='copy' in args)
|