Skip to content

Owo Sugiana / maintenance

Go to a project
Commit 13fdd952 by Owo Sugiana
Options

Jumlah backup diatur di konfigurasi

1 parent fdc1650c
Inline Side-by-side
Showing 8 changed files with 170 additions and 47 deletions
.gitignore
test* test*
__pycache__
CHANGES.txt
0.2 24-12-2020
--------------
- Jumlah backup bisa diatur di konfigurasi
- Log yang lebih fleksibel melalui konfigurasi
- Cluster mirror tidak di-VACUUM
0.1 29-8-2020 0.1 29-8-2020
------------- -------------
- Kali pertama - Kali pertama
README.rst
...@@ -36,8 +36,25 @@ Buat file konfigurasi ``maintenance.ini``:: ...@@ -36,8 +36,25 @@ Buat file konfigurasi ``maintenance.ini``::
# Lokasi direktori hasil pg_dump # Lokasi direktori hasil pg_dump
database_backup_dir = /var/backups/pg database_backup_dir = /var/backups/pg
log_format = %(asctime)s %(levelname)s %(message)s # Jumlah masing-masing database yang tersimpan
log_file = /var/log/maintenance.log database_rotate = 5
[formatter_generic]
format = %(asctime)s %(levelname)s %(message)s
[handler_console]
class = StreamHandler
stream = sys.stdout
formatter = generic
[handler_file]
class = FileHandler
filename = /var/log/maintenance.log
formatter = generic
[logger_root]
handlers = console, file
level = DEBUG
Konfigurasi ini dibuat untuk Debian 8 ke atas yaitu yang sudah menerapkan Konfigurasi ini dibuat untuk Debian 8 ke atas yaitu yang sudah menerapkan
``systemd``. Sesuaikan baris ``database`` yang berisi nama database yang akan ``systemd``. Sesuaikan baris ``database`` yang berisi nama database yang akan
......
maintenance.ini
...@@ -25,5 +25,22 @@ database = ...@@ -25,5 +25,22 @@ database =
# Lokasi direktori hasil pg_dump # Lokasi direktori hasil pg_dump
database_backup_dir = /var/backups/pg database_backup_dir = /var/backups/pg
log_format = %(asctime)s %(levelname)s %(message)s # Jumlah masing-masing database yang tersimpan
log_file = /var/log/maintenance.log database_rotate = 5
[formatter_generic]
format = %(asctime)s %(levelname)s %(message)s
[handler_console]
class = StreamHandler
stream = sys.stdout
formatter = generic
[handler_file]
class = FileHandler
filename = /var/log/maintenance.log
formatter = generic
[logger_root]
handlers = console, file
level = DEBUG
maintenance/__init__.py
...@@ -9,17 +9,16 @@ from subprocess import ( ...@@ -9,17 +9,16 @@ from subprocess import (
PIPE, PIPE,
call, call,
) )
from configparser import ( from configparser import ConfigParser
ConfigParser,
RawConfigParser,
)
from argparse import ArgumentParser from argparse import ArgumentParser
import shutil from .pg_conf import Reader
from .file_rotate import FileRotate
from .logger import setup_logging
RE_COUNT_ROW = re.compile(r'(\d*) row') RE_COUNT_ROW = re.compile(r'(\d*) row')
log = logging.getLogger(sys.argv[0])
conf = ConfigParser() conf = ConfigParser()
log = logging.getLogger()
def get_tmp_files(dir_name, db_name): def get_tmp_files(dir_name, db_name):
...@@ -85,17 +84,27 @@ def pg_port(conf_file): ...@@ -85,17 +84,27 @@ def pg_port(conf_file):
return value return value
def is_mirror(pg_conf):
r = Reader(pg_conf)
mirror_conf = os.path.join(r['data_directory'], 'recovery.conf')
return os.path.exists(mirror_conf)
# Pastikan database server hanya bisa diakses oleh user postgres secara local # Pastikan database server hanya bisa diakses oleh user postgres secara local
# Abaikan bila cluster adalah mirror
def pg_local_conf(): def pg_local_conf():
pg_conf_dir = conf.get('main', 'pg_conf_dir') pg_conf_dir = conf.get('main', 'pg_conf_dir')
for version in os.listdir(pg_conf_dir): for version in os.listdir(pg_conf_dir):
version_dir = os.path.join(pg_conf_dir, version) conf_dir = os.path.join(pg_conf_dir, version)
for cluster in os.listdir(version_dir): for cluster in os.listdir(conf_dir):
cluster_dir = os.path.join(version_dir, cluster) cluster_dir = os.path.join(conf_dir, cluster)
pg_conf = os.path.join(cluster_dir, 'postgresql.conf')
if is_mirror(pg_conf):
continue
pg_hba_file = os.path.join(cluster_dir, 'pg_hba.conf') pg_hba_file = os.path.join(cluster_dir, 'pg_hba.conf')
pg_hba_bak_file = pg_hba_file + '.orig' pg_hba_bak_file = pg_hba_file + '.orig'
if not os.path.exists(pg_hba_bak_file): if not os.path.exists(pg_hba_bak_file):
shutil.copy(pg_hba_file, pg_hba_bak_file) os.rename(pg_hba_file, pg_hba_bak_file)
version_float = float(version) version_float = float(version)
if version_float <= 8.4: if version_float <= 8.4:
line = 'local all postgres ident' line = 'local all postgres ident'
...@@ -115,8 +124,9 @@ def pg_original_conf(): ...@@ -115,8 +124,9 @@ def pg_original_conf():
cluster_dir = os.path.join(version_dir, cluster) cluster_dir = os.path.join(version_dir, cluster)
pg_hba_file = os.path.join(cluster_dir, 'pg_hba.conf') pg_hba_file = os.path.join(cluster_dir, 'pg_hba.conf')
pg_hba_bak_file = pg_hba_file + '.orig' pg_hba_bak_file = pg_hba_file + '.orig'
if not os.path.exists(pg_hba_bak_file):
continue
os.rename(pg_hba_bak_file, pg_hba_file) os.rename(pg_hba_bak_file, pg_hba_file)
chown_postgres(pg_hba_file)
def is_db_exists(version, port, db_name): def is_db_exists(version, port, db_name):
...@@ -147,7 +157,7 @@ def chown_postgres(filename): ...@@ -147,7 +157,7 @@ def chown_postgres(filename):
def vacuum(version, port, db_name): def vacuum(version, port, db_name):
psql_bin = pg_bin(version, 'psql') psql_bin = pg_bin(version, 'psql')
sql = 'VACUUM FULL VERBOSE ANALYZE' sql = 'VACUUM FULL VERBOSE ANALYZE'
cmd = f'{psql_bin} -p {port} -c "{sql}"' cmd = f'{psql_bin} -p {port} {db_name} -c "{sql}"'
run_as_postgres(cmd) run_as_postgres(cmd)
...@@ -158,9 +168,8 @@ def pg_bin(version, cmd): ...@@ -158,9 +168,8 @@ def pg_bin(version, cmd):
def pg_dump(version, port, db_name, dir_name): def pg_dump(version, port, db_name, dir_name):
backup_file = os.path.join(dir_name, db_name) + '.pg' backup_file = os.path.join(dir_name, db_name) + '.pg'
if os.path.exists(backup_file) and os.stat(backup_file).st_size: fr = FileRotate(backup_file, conf.getint('main', 'database_rotate'))
old_file = backup_file + '.old' fr.run()
os.rename(backup_file, old_file)
pg_dump_bin = pg_bin(version, 'pg_dump') pg_dump_bin = pg_bin(version, 'pg_dump')
tmp_file = os.path.join(dir_name, db_name) + '_' tmp_file = os.path.join(dir_name, db_name) + '_'
cmd = f'{pg_dump_bin} -p {port} -Fc {db_name} | split -b 1G - {tmp_file}' cmd = f'{pg_dump_bin} -p {port} -Fc {db_name} | split -b 1G - {tmp_file}'
...@@ -182,20 +191,21 @@ def pg_backup(): ...@@ -182,20 +191,21 @@ def pg_backup():
db_list = conf.get('main', 'database').split() db_list = conf.get('main', 'database').split()
pg_conf_dir = conf.get('main', 'pg_conf_dir') pg_conf_dir = conf.get('main', 'pg_conf_dir')
for version in os.listdir(pg_conf_dir): for version in os.listdir(pg_conf_dir):
version_dir = os.path.join(pg_conf_dir, version) version_conf_dir = os.path.join(pg_conf_dir, version)
backup_dir = os.path.join(base_backup_dir, version) version_backup_dir = os.path.join(base_backup_dir, version)
mkdir(backup_dir) mkdir(version_backup_dir)
for cluster in os.listdir(version_dir): for cluster in os.listdir(version_conf_dir):
backup_dir = os.path.join(backup_dir, cluster) cluster_conf_dir = os.path.join(version_conf_dir, cluster)
mkdir(backup_dir) cluster_backup_dir = os.path.join(version_backup_dir, cluster)
cluster_dir = os.path.join(version_dir, cluster) mkdir(cluster_backup_dir)
postgresql_conf = os.path.join(cluster_dir, 'postgresql.conf') pg_conf = os.path.join(cluster_conf_dir, 'postgresql.conf')
port = pg_port(postgresql_conf) port = pg_port(pg_conf)
for db_name in db_list: for db_name in db_list:
if not is_db_exists(version, port, db_name): if not is_db_exists(version, port, db_name):
continue continue
vacuum(version, port, db_name) if not is_mirror(pg_conf):
pg_dump(version, port, db_name, backup_dir) vacuum(version, port, db_name)
pg_dump(version, port, db_name, cluster_backup_dir)
def mkdir(name): def mkdir(name):
...@@ -211,26 +221,10 @@ def get_option(argv): ...@@ -211,26 +221,10 @@ def get_option(argv):
return parser.parse_args(argv) return parser.parse_args(argv)
def set_log(conf_file):
log_conf = RawConfigParser()
log_conf.read(conf_file)
log_cf = log_conf['main']
log_format = log_cf['log_format']
log_file = log_cf['log_file']
formatter = logging.Formatter(log_format)
file_handler = logging.FileHandler(log_file)
file_handler.setFormatter(formatter)
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setFormatter(formatter)
log.setLevel(logging.DEBUG)
log.addHandler(file_handler)
log.addHandler(console_handler)
def main(argv=sys.argv[1:]): def main(argv=sys.argv[1:]):
option = get_option(argv) option = get_option(argv)
conf_file = option.config conf_file = option.config
set_log(conf_file) setup_logging(conf_file)
conf.read(conf_file) conf.read(conf_file)
run_event('before') run_event('before')
if option.stop_daemons_and_exit: if option.stop_daemons_and_exit:
......
maintenance/file_rotate.py 0 → 100644
import os
import logging
log = logging.getLogger()
class FileRotate:
def __init__(self, filename, count=5):
self.filename = filename
self.count = count
def run(self):
numbers = list(range(1, self.count+1))
numbers.reverse()
for b in numbers:
file_b = f'{self.filename}.{b}'
if b == 1:
file_a = self.filename
else:
a = b - 1
file_a = f'{self.filename}.{a}'
if not os.path.exists(file_a):
continue
if os.stat(file_a).st_size:
self.on_rotate(file_a, file_b)
else:
log.debug(f'{file_a} kosong maka tidak perlu diamankan')
def on_rotate(self, file_a, file_b):
os.rename(file_a, file_b)
log.debug(f'mv {file_a} {file_b}')
maintenance/logger.py 0 → 100644
from configparser import RawConfigParser
import logging
import logging.config
def setup_logging(conf_file):
conf = RawConfigParser()
conf.read(conf_file)
d = {
'version': 1,
'formatters': {},
'handlers': {},
'loggers': {},
}
for section in conf.sections():
if section.find('formatter_') == 0:
name = section.split('formatter_')[1]
data = {'format': conf.get(section, 'format')}
d['formatters'][name] = data
elif section.find('handler_') == 0:
name = section.split('handler_')[1]
data = {'formatter': conf.get(section, 'formatter')}
data['class'] = 'logging.' + conf.get(section, 'class')
if conf.has_option(section, 'stream'):
data['stream'] = 'ext://' + conf.get(section, 'stream')
else:
data['filename'] = conf.get(section, 'filename')
d['handlers'][name] = data
elif section.find('logger_') == 0:
name = section.split('logger_')[1]
if name == 'root':
name = ''
data = {'level': conf.get(section, 'level')}
value = conf.get(section, 'handlers')
data['handlers'] = [x.strip() for x in value.split(',')]
d['loggers'][name] = data
logging.config.dictConfig(d)
maintenance/pg_conf.py 0 → 100644
class Reader:
def __init__(self, conf_file):
self.conf = dict()
with open(conf_file) as f:
for line in f.readlines():
s = line.strip()
if not s:
continue
if s[0] == '#':
continue
t = s.split('#')
t = t[0].split('=')
key = t[0].strip()
value = '='.join(t[1:])
value = value.strip()
self.conf[key] = value.strip("'")
def __getitem__(self, key):
return self.conf[key]
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!