Jumlah backup diatur di konfigurasi

.gitignore

 test*
+__pycache__

CHANGES.txt

+0.2 24-12-2020
+--------------
+- Jumlah backup bisa diatur di konfigurasi
+- Log yang lebih fleksibel melalui konfigurasi
+- Cluster mirror tidak di-VACUUM
+
 0.1 29-8-2020
 -------------
 - Kali pertama

README.rst

@@ -36,8 +36,25 @@ Buat file konfigurasi ``maintenance.ini``::
     # Lokasi direktori hasil pg_dump
     database_backup_dir = /var/backups/pg
 
-    log_format = %(asctime)s %(levelname)s %(message)s
-    log_file = /var/log/maintenance.log
+    # Jumlah masing-masing database yang tersimpan
+    database_rotate = 5
+
+    [formatter_generic]
+    format = %(asctime)s %(levelname)s %(message)s
+
+    [handler_console]
+    class = StreamHandler
+    stream = sys.stdout
+    formatter = generic
+
+    [handler_file]
+    class = FileHandler
+    filename = /var/log/maintenance.log
+    formatter = generic
+
+    [logger_root]
+    handlers = console, file
+    level = DEBUG
 
 Konfigurasi ini dibuat untuk Debian 8 ke atas yaitu yang sudah menerapkan
 ``systemd``. Sesuaikan baris ``database`` yang berisi nama database yang akan

maintenance.ini

@@ -25,5 +25,22 @@ database =
 # Lokasi direktori hasil pg_dump
 database_backup_dir = /var/backups/pg
 
-log_format = %(asctime)s %(levelname)s %(message)s
-log_file = /var/log/maintenance.log
+# Jumlah masing-masing database yang tersimpan
+database_rotate = 5
+
+[formatter_generic]
+format = %(asctime)s %(levelname)s %(message)s
+
+[handler_console]
+class = StreamHandler
+stream = sys.stdout
+formatter = generic
+
+[handler_file]
+class = FileHandler
+filename = /var/log/maintenance.log
+formatter = generic
+
+[logger_root]
+handlers = console, file
+level = DEBUG

maintenance/__init__.py

@@ -9,17 +9,16 @@ from subprocess import (
     PIPE,
     call,
     )
-from configparser import (
-    ConfigParser,
-    RawConfigParser,
-    )
+from configparser import ConfigParser
 from argparse import ArgumentParser
-import shutil
+from .pg_conf import Reader
+from .file_rotate import FileRotate
+from .logger import setup_logging
 
 
 RE_COUNT_ROW = re.compile(r'(\d*) row')
-log = logging.getLogger(sys.argv[0])
 conf = ConfigParser()
+log = logging.getLogger()
 
 
 def get_tmp_files(dir_name, db_name):
@@ -85,17 +84,27 @@ def pg_port(conf_file):
                 return value
 
 
+def is_mirror(pg_conf):
+    r = Reader(pg_conf)
+    mirror_conf = os.path.join(r['data_directory'], 'recovery.conf')
+    return os.path.exists(mirror_conf)
+
+
 # Pastikan database server hanya bisa diakses oleh user postgres secara local
+# Abaikan bila cluster adalah mirror
 def pg_local_conf():
     pg_conf_dir = conf.get('main', 'pg_conf_dir')
     for version in os.listdir(pg_conf_dir):
-        version_dir = os.path.join(pg_conf_dir, version)
-        for cluster in os.listdir(version_dir):
-            cluster_dir = os.path.join(version_dir, cluster)
+        conf_dir = os.path.join(pg_conf_dir, version)
+        for cluster in os.listdir(conf_dir):
+            cluster_dir = os.path.join(conf_dir, cluster)
+            pg_conf = os.path.join(cluster_dir, 'postgresql.conf')
+            if is_mirror(pg_conf):
+                continue
             pg_hba_file = os.path.join(cluster_dir, 'pg_hba.conf')
             pg_hba_bak_file = pg_hba_file + '.orig'
             if not os.path.exists(pg_hba_bak_file):
-                shutil.copy(pg_hba_file, pg_hba_bak_file)
+                os.rename(pg_hba_file, pg_hba_bak_file)
             version_float = float(version)
             if version_float <= 8.4:
                 line = 'local all postgres ident'
@@ -115,8 +124,9 @@ def pg_original_conf():
             cluster_dir = os.path.join(version_dir, cluster)
             pg_hba_file = os.path.join(cluster_dir, 'pg_hba.conf')
             pg_hba_bak_file = pg_hba_file + '.orig'
+            if not os.path.exists(pg_hba_bak_file):
+                continue
             os.rename(pg_hba_bak_file, pg_hba_file)
-            chown_postgres(pg_hba_file)
 
 
 def is_db_exists(version, port, db_name):
@@ -147,7 +157,7 @@ def chown_postgres(filename):
 def vacuum(version, port, db_name):
     psql_bin = pg_bin(version, 'psql')
     sql = 'VACUUM FULL VERBOSE ANALYZE'
-    cmd = f'{psql_bin} -p {port} -c "{sql}"'
+    cmd = f'{psql_bin} -p {port} {db_name} -c "{sql}"'
     run_as_postgres(cmd)
 
 
@@ -158,9 +168,8 @@ def pg_bin(version, cmd):
 
 def pg_dump(version, port, db_name, dir_name):
     backup_file = os.path.join(dir_name, db_name) + '.pg'
-    if os.path.exists(backup_file) and os.stat(backup_file).st_size:
-        old_file = backup_file + '.old'
-        os.rename(backup_file, old_file)
+    fr = FileRotate(backup_file, conf.getint('main', 'database_rotate'))
+    fr.run()
     pg_dump_bin = pg_bin(version, 'pg_dump')
     tmp_file = os.path.join(dir_name, db_name) + '_'
     cmd = f'{pg_dump_bin} -p {port} -Fc {db_name} | split -b 1G - {tmp_file}'
@@ -182,20 +191,21 @@ def pg_backup():
     db_list = conf.get('main', 'database').split()
     pg_conf_dir = conf.get('main', 'pg_conf_dir')
     for version in os.listdir(pg_conf_dir):
-        version_dir = os.path.join(pg_conf_dir, version)
-        backup_dir = os.path.join(base_backup_dir, version)
-        mkdir(backup_dir)
-        for cluster in os.listdir(version_dir):
-            backup_dir = os.path.join(backup_dir, cluster)
-            mkdir(backup_dir)
-            cluster_dir = os.path.join(version_dir, cluster)
-            postgresql_conf = os.path.join(cluster_dir, 'postgresql.conf')
-            port = pg_port(postgresql_conf)
+        version_conf_dir = os.path.join(pg_conf_dir, version)
+        version_backup_dir = os.path.join(base_backup_dir, version)
+        mkdir(version_backup_dir)
+        for cluster in os.listdir(version_conf_dir):
+            cluster_conf_dir = os.path.join(version_conf_dir, cluster)
+            cluster_backup_dir = os.path.join(version_backup_dir, cluster)
+            mkdir(cluster_backup_dir)
+            pg_conf = os.path.join(cluster_conf_dir, 'postgresql.conf')
+            port = pg_port(pg_conf)
             for db_name in db_list:
                 if not is_db_exists(version, port, db_name):
                     continue
+                if not is_mirror(pg_conf):
                     vacuum(version, port, db_name)
-                pg_dump(version, port, db_name, backup_dir)
+                pg_dump(version, port, db_name, cluster_backup_dir)
 
 
 def mkdir(name):
@@ -211,26 +221,10 @@ def get_option(argv):
     return parser.parse_args(argv)
 
 
-def set_log(conf_file):
-    log_conf = RawConfigParser()
-    log_conf.read(conf_file)
-    log_cf = log_conf['main']
-    log_format = log_cf['log_format']
-    log_file = log_cf['log_file']
-    formatter = logging.Formatter(log_format)
-    file_handler = logging.FileHandler(log_file)
-    file_handler.setFormatter(formatter)
-    console_handler = logging.StreamHandler(sys.stdout)
-    console_handler.setFormatter(formatter)
-    log.setLevel(logging.DEBUG)
-    log.addHandler(file_handler)
-    log.addHandler(console_handler)
-
-
 def main(argv=sys.argv[1:]):
     option = get_option(argv)
     conf_file = option.config
-    set_log(conf_file)
+    setup_logging(conf_file)
     conf.read(conf_file)
     run_event('before')
     if option.stop_daemons_and_exit:

maintenance/file_rotate.py

+import os
+import logging
+
+
+log = logging.getLogger()
+
+
+class FileRotate:
+    def __init__(self, filename, count=5):
+        self.filename = filename
+        self.count = count
+
+    def run(self):
+        numbers = list(range(1, self.count+1))
+        numbers.reverse()
+        for b in numbers:
+            file_b = f'{self.filename}.{b}'
+            if b == 1:
+                file_a = self.filename
+            else:
+                a = b - 1
+                file_a = f'{self.filename}.{a}'
+            if not os.path.exists(file_a):
+                continue
+            if os.stat(file_a).st_size:
+                self.on_rotate(file_a, file_b)
+            else:
+                log.debug(f'{file_a} kosong maka tidak perlu diamankan')
+
+    def on_rotate(self, file_a, file_b):
+        os.rename(file_a, file_b)
+        log.debug(f'mv {file_a} {file_b}')

maintenance/logger.py

+from configparser import RawConfigParser
+import logging
+import logging.config
+
+
+def setup_logging(conf_file):
+    conf = RawConfigParser()
+    conf.read(conf_file)
+    d = {
+        'version': 1,
+        'formatters': {},
+        'handlers': {},
+        'loggers': {},
+        }
+    for section in conf.sections():
+        if section.find('formatter_') == 0:
+            name = section.split('formatter_')[1]
+            data = {'format': conf.get(section, 'format')}
+            d['formatters'][name] = data
+        elif section.find('handler_') == 0:
+            name = section.split('handler_')[1]
+            data = {'formatter': conf.get(section, 'formatter')}
+            data['class'] = 'logging.' + conf.get(section, 'class')
+            if conf.has_option(section, 'stream'):
+                data['stream'] = 'ext://' + conf.get(section, 'stream')
+            else:
+                data['filename'] = conf.get(section, 'filename')
+            d['handlers'][name] = data
+        elif section.find('logger_') == 0:
+            name = section.split('logger_')[1]
+            if name == 'root':
+                name = ''
+            data = {'level': conf.get(section, 'level')}
+            value = conf.get(section, 'handlers')
+            data['handlers'] = [x.strip() for x in value.split(',')]
+            d['loggers'][name] = data
+    logging.config.dictConfig(d)

maintenance/pg_conf.py

+class Reader:
+    def __init__(self, conf_file):
+        self.conf = dict()
+        with open(conf_file) as f:
+            for line in f.readlines():
+                s = line.strip()
+                if not s:
+                    continue
+                if s[0] == '#':
+                    continue
+                t = s.split('#')
+                t = t[0].split('=')
+                key = t[0].strip()
+                value = '='.join(t[1:])
+                value = value.strip()
+                self.conf[key] = value.strip("'")
+
+    def __getitem__(self, key):
+        return self.conf[key]