1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
|
# This file is part of rhizi, a collaborative knowledge graph editor.
# Copyright (C) 2014-2015 Rhizi
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#!/usr/bin/python2.7
import click
from datetime import datetime
from glob import glob
import os
from subprocess import Popen
from os.path import basename, dirname, exists
import re
import shutil
# Set by command line arguments, defaults below Click.command()
root = None
docker_container_id = None
neo4j_shell = None
debug = True
backup_template = 'backup-%s.dump'
backup_glob = backup_template % '*'
backup_re = re.compile(backup_template % '(\d\d\d\d)(\d\d)(\d\d)')
DAILY_KEPT = 7 # should be >= 7
WEEKLY_KEPT = 5 # should be >= 5
def unlink(filename):
if debug:
print("unlinking %s")
os.unlink(filename)
def makedirs(path):
if exists(path):
return
if debug:
print("makedirs %s" % path)
os.makedirs(path)
def copy(src, dst):
if debug:
print("copy %s %s" % (src, dst))
if not exists(src):
print("bug: %s doesn't exist") % src
raise SystemExit
dst_path = dirname(dst)
if not exists(dst_path):
makedirs(dst_path)
shutil.copy(src, dst)
def neo4jshell(command, stdout):
args = [neo4j_shell, '-c', command]
if docker_container_id is not None:
args = ['docker', 'exec', docker_container_id] + args
if debug:
print("executing %s (%r)" % (' '.join(args), args))
Popen(args=args, stdout=open(stdout, 'w+')).wait()
def next_daily_filename():
filename = os.path.join(daily_root(), backup_template % datetime.now().strftime('%Y%m%d'))
return filename
def daily_root():
return os.path.join(root, 'daily')
def weekly_root():
return os.path.join(root, 'weekly')
def monthly_root():
return os.path.join(root, 'monthly')
def backup_year_month_day(filename):
return map(int, backup_re.match(basename(filename)).groups())
def to_weekly(filename):
year, month, day = backup_year_month_day(filename)
_, (wkly_year, wkly_month, wkly_day) = latest_weekly()
# import pdb; pdb.set_trace()
delta = datetime(year, month, day) - datetime(wkly_year, wkly_month, wkly_day)
return delta.days >= 7
def to_monthly(filename):
""" return True if filename is newer by a month from the newest monthly """
year, month, day = backup_year_month_day(filename)
_, (monthly_year, monthly_month, monthly_day) = latest_monthly()
if monthly_year < year:
month += 12
return month > monthly_month and day >= monthly_day
def latest_weekly():
return latest(weekly_root())
def latest_monthly():
return latest(monthly_root())
def latest(subdir):
"""
returns filename, (year, month, day)
"""
filenames = glob(os.path.join(subdir, backup_glob))
if len(filenames) == 0:
return 'no_such_file', (1848, 1, 1)
return max((fname, backup_year_month_day(fname)) for fname in filenames)
def sorted_backups(subdir):
def cmp_files(fa, fb):
ta = os.stat(fa).st_mtime
tb = os.stat(fb).st_mtime
if ta == tb:
return 0
if ta < tb:
return -1
return 1
return sorted(glob(os.path.join(weekly_root(), backup_glob)), cmp=cmp_files)
def create_daily_backup():
filename = next_daily_filename()
makedirs(dirname(filename))
neo4jshell("dump", stdout=filename)
return filename
@click.command()
@click.option('--root', 'cli_root', default='/var/lib/rhizi/backup',
type=click.Path(exists=True))
@click.option('--docker', 'cli_docker_container_id', default=None)
@click.option('--neo4j-shell', 'cli_neo4j_shell', default='neo4j-shell')
def backup(cli_root, cli_docker_container_id, cli_neo4j_shell):
global root
global docker_container_id
global neo4j_shell
root = cli_root
docker_container_id = cli_docker_container_id
neo4j_shell = cli_neo4j_shell
# create the daily backup
filename = create_daily_backup()
# copy out to weekly if a week passed since last weekly, otherwise remove excess
if to_weekly(filename):
copy(filename, os.path.join(weekly_root(), basename(filename)))
for filename in sorted_backups(daily_root())[DAILY_KEPT:]:
unlink(filename)
# rotate out to monthly if a month passed since last monthly, otherwise remove excess
if to_monthly(filename):
copy(filename, os.path.join(monthly_root(), basename(filename)))
for filename in sorted_backups(weekly_root())[WEEKLY_KEPT:]:
unlink(filename)
if __name__ == '__main__':
backup()
|
