Kali pertama

.gitignore

+test*

CHANGES.txt

+0.1 29-8-2020
+-------------
+- Kali pertama

README.rst

+Maintenance
+===========
+Script ini untuk vacuum dan backup database PostgreSQL. Berikut cara
+memasangnya::
+
+    $ python3 -m venv env
+    $ env/bin/pip install --upgrade pip
+    $ env/bin/pip install git+https://git.opensipkd.com/sugiana/maintenance
+
+Buat file konfigurasi ``maintenance.ini``::
+
+    [main]
+    # Perintah yang dijalankan sebelum pg_dump 
+    before =
+        /usr/sbin/ntpdate -u ntp.ubuntu.com
+        systemctl stop cron.service
+        systemctl stop nginx.service
+
+    # Perintah yang dijalankan setelah pg_dump
+    after =
+        systemctl start nginx.service
+        systemctl start cron.service
+
+    pg_stop = systemctl stop postgresql.service
+    pg_start = systemctl start postgresql.service
+
+    pg_lib_dir = /usr/lib/postgresql
+    pg_conf_dir = /etc/postgresql
+
+    # Daftar database yang akan di-vacuum dan di-pg_dump
+    database =
+        pbb
+        bphtb
+        pad
+
+    # Lokasi direktori hasil pg_dump
+    database_backup_dir = /var/backups/pg
+
+    log_format = %(asctime)s %(levelname)s %(message)s
+    log_file = /var/log/maintenance.log
+
+Konfigurasi ini dibuat untuk Debian 8 ke atas yaitu yang sudah menerapkan
+``systemd``. Sesuaikan baris ``database`` yang berisi nama database yang akan
+di-backup. Baris ``before`` biasanya diisi dengan perintah untuk menghentikan
+daemon yang menggunakan database, sedangkan baris ``after`` untuk
+menghidupkannya kembali setelah proses backup selesai.
+
+Selanjutnya jalankan sebagai root::
+
+    $ sudo env/bin/maintenance maintenance.ini
+
+Daftarkan di cron pada jam 1:45 saat dimana kegiatan transaksi jarang
+dilakukan. Contohnya di ``/etc/cron.d/maintenance`` berikut ini::
+
+    MAILTO=""
+    45 1 * * * root cd /home/sugiana/2020/08 && env/bin/maintenance maintenance.ini >/dev/null 2>&1
+
+Semoga berhasil.

maintenance.ini

+[main]
+# Perintah yang dijalankan sebelum pg_dump 
+before =
+    /usr/sbin/ntpdate -u ntp.ubuntu.com
+    systemctl stop cron.service
+    systemctl stop nginx.service
+
+# Perintah yang dijalankan setelah pg_dump
+after =
+    systemctl start nginx.service
+    systemctl start cron.service
+
+pg_stop = systemctl stop postgresql.service
+pg_start = systemctl start postgresql.service
+
+pg_lib_dir = /usr/lib/postgresql
+pg_conf_dir = /etc/postgresql
+
+# Daftar database yang akan di-vacuum dan di-pg_dump
+database =
+    pbb
+    bphtb
+    pad
+
+# Lokasi direktori hasil pg_dump
+database_backup_dir = /var/backups/pg
+
+log_format = %(asctime)s %(levelname)s %(message)s
+log_file = /var/log/maintenance.log

maintenance/__init__.py

+import os
+import sys
+import re
+from time import sleep
+import logging
+from string import ascii_lowercase
+from subprocess import (
+    Popen,
+    PIPE,
+    call,
+    )
+from configparser import (
+    ConfigParser,
+    RawConfigParser,
+    )
+from argparse import ArgumentParser
+import shutil
+
+
+RE_COUNT_ROW = re.compile(r'(\d*) row')
+log = logging.getLogger(sys.argv[0])
+conf = ConfigParser()
+
+
+def get_tmp_files(dir_name, db_name):
+    r = []
+    for ch1 in ascii_lowercase:
+        for ch2 in ascii_lowercase:
+            suffix = ch1 + ch2
+            filename = '_'.join([db_name, suffix])
+            filename = os.path.join(dir_name, filename)
+            if not os.path.exists(filename):
+                return r
+            r.append(filename)
+    return r
+
+
+def run(command):
+    log.info(command)
+    if os.system(command) != 0:
+        sys.exit()
+
+
+def run_list(cmd):
+    log.info(' '.join(cmd))
+    if call(cmd) != 0:
+        sys.exit()
+
+
+def run_event(event):
+    scripts = conf.get('main', event)
+    for script in scripts.splitlines():
+        script = script.strip()
+        if not script:
+            continue
+        run(script)
+
+
+def pg_service(todo):
+    var = 'pg_' + todo
+    cmd = conf.get('main', var)
+    run(cmd)
+
+
+def pg_stop():
+    pg_service('stop')
+
+
+def pg_start():
+    pg_service('start')
+    # Beri waktu database server untuk up
+    sleep(5)
+
+
+def pg_port(conf_file):
+    with open(conf_file) as f:
+        for line in f.readlines():
+            s = line.strip()
+            if not s or s[0] == '#':
+                continue
+            t = s.split('=')
+            key = t[0].strip()
+            value = t[1].split('#')[0].strip()
+            if key == 'port':
+                return value
+
+
+# Pastikan database server hanya bisa diakses oleh user postgres secara local
+def pg_local_conf():
+    pg_conf_dir = conf.get('main', 'pg_conf_dir')
+    for version in os.listdir(pg_conf_dir):
+        version_dir = os.path.join(pg_conf_dir, version)
+        for cluster in os.listdir(version_dir):
+            cluster_dir = os.path.join(version_dir, cluster)
+            pg_hba_file = os.path.join(cluster_dir, 'pg_hba.conf')
+            pg_hba_bak_file = pg_hba_file + '.orig'
+            if not os.path.exists(pg_hba_bak_file):
+                shutil.copy(pg_hba_file, pg_hba_bak_file)
+            version_float = float(version)
+            if version_float <= 8.4:
+                line = 'local all postgres ident'
+            elif version_float > 8.4 and version_float < 9:
+                line = 'local all postgres ident sameuser'
+            else:
+                line = 'local all postgres peer'
+            with open(pg_hba_file, 'w') as f:
+                f.write(line)
+
+
+def pg_original_conf():
+    pg_conf_dir = conf.get('main', 'pg_conf_dir')
+    for version in os.listdir(pg_conf_dir):
+        version_dir = os.path.join(pg_conf_dir, version)
+        for cluster in os.listdir(version_dir):
+            cluster_dir = os.path.join(version_dir, cluster)
+            pg_hba_file = os.path.join(cluster_dir, 'pg_hba.conf')
+            pg_hba_bak_file = pg_hba_file + '.orig'
+            os.rename(pg_hba_bak_file, pg_hba_file)
+            chown_postgres(pg_hba_file)
+
+
+def is_db_exists(version, port, db_name):
+    psql_bin = pg_bin(version, 'psql')
+    sql = f"SELECT 1 FROM pg_database WHERE datname='{db_name}'"
+    cmd_psql = f'{psql_bin} -p {port} -c "{sql}"'
+    cmd = ['su', '-', 'postgres', '-c', cmd_psql]
+    p = Popen(cmd, stdout=PIPE)
+    out, err = p.communicate()
+    s = out.decode('utf-8')
+    match = RE_COUNT_ROW.search(s)
+    if match:
+        count = match.group(1)
+        return int(count)
+    sys.exit()
+
+
+def run_as_postgres(cmd):
+    cmd = ['su', '-', 'postgres', '-c', cmd]
+    run_list(cmd)
+
+
+def chown_postgres(filename):
+    cmd = ['chown', 'postgres.postgres', filename]
+    run_list(cmd)
+
+
+def vacuum(version, port, db_name):
+    psql_bin = pg_bin(version, 'psql')
+    sql = 'VACUUM FULL VERBOSE ANALYZE'
+    cmd = f'{psql_bin} -p {port} -c "{sql}"'
+    run_as_postgres(cmd)
+
+
+def pg_bin(version, cmd):
+    pg_lib_dir = conf.get('main', 'pg_lib_dir')
+    return os.path.join(pg_lib_dir, version, 'bin', cmd)
+
+
+def pg_dump(version, port, db_name, dir_name):
+    backup_file = os.path.join(dir_name, db_name) + '.pg'
+    if os.path.exists(backup_file) and os.stat(backup_file).st_size:
+        old_file = backup_file + '.old'
+        os.rename(backup_file, old_file)
+    pg_dump_bin = pg_bin(version, 'pg_dump')
+    tmp_file = os.path.join(dir_name, db_name) + '_'
+    cmd = f'{pg_dump_bin} -p {port} -Fc {db_name} | split -b 1G - {tmp_file}'
+    run_as_postgres(cmd)
+    tmp_files = get_tmp_files(dir_name, db_name)
+    if not tmp_files:
+        return
+    files = ' '.join(tmp_files)
+    run(f'cat {files} > {backup_file}')
+    chown_postgres(backup_file)
+    for filename in tmp_files:
+        os.remove(filename)
+
+
+# http://postgresql.1045698.n5.nabble.com/large-database-problems-with-pg-dump-and-pg-restore-td3236910.html
+def pg_backup():
+    base_backup_dir = conf.get('main', 'database_backup_dir')
+    mkdir(base_backup_dir)
+    db_list = conf.get('main', 'database').split()
+    pg_conf_dir = conf.get('main', 'pg_conf_dir')
+    for version in os.listdir(pg_conf_dir):
+        version_dir = os.path.join(pg_conf_dir, version)
+        backup_dir = os.path.join(base_backup_dir, version)
+        mkdir(backup_dir)
+        for cluster in os.listdir(version_dir):
+            backup_dir = os.path.join(backup_dir, cluster)
+            mkdir(backup_dir)
+            cluster_dir = os.path.join(version_dir, cluster)
+            postgresql_conf = os.path.join(cluster_dir, 'postgresql.conf')
+            port = pg_port(postgresql_conf)
+            for db_name in db_list:
+                if not is_db_exists(version, port, db_name):
+                    continue
+                vacuum(version, port, db_name)
+                pg_dump(version, port, db_name, backup_dir)
+
+
+def mkdir(name):
+    if not os.path.exists(name):
+        os.mkdir(name)
+        chown_postgres(name)
+
+
+def get_option(argv):
+    parser = ArgumentParser()
+    parser.add_argument('config')
+    parser.add_argument('--stop-daemons-and-exit')
+    return parser.parse_args(argv)
+
+
+def set_log(conf_file):
+    log_conf = RawConfigParser()
+    log_conf.read(conf_file)
+    log_cf = log_conf['main']
+    log_format = log_cf['log_format']
+    log_file = log_cf['log_file']
+    formatter = logging.Formatter(log_format)
+    file_handler = logging.FileHandler(log_file)
+    file_handler.setFormatter(formatter)
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_handler.setFormatter(formatter)
+    log.setLevel(logging.DEBUG)
+    log.addHandler(file_handler)
+    log.addHandler(console_handler)
+
+
+def main(argv=sys.argv[1:]):
+    option = get_option(argv)
+    conf_file = option.config
+    set_log(conf_file)
+    conf.read(conf_file)
+    run_event('before')
+    if option.stop_daemons_and_exit:
+        return
+    pg_stop()
+    pg_local_conf()
+    pg_start()
+    pg_backup()
+    pg_stop()
+    pg_original_conf()
+    pg_start()
+    run_event('after')

setup.py

+import os
+from setuptools import (
+    setup,
+    find_packages,
+    )
+
+
+here = os.path.abspath(os.path.dirname(__file__))
+with open(os.path.join(here, 'README.rst')) as f:
+    README = f.read()
+with open(os.path.join(here, 'CHANGES.txt')) as f:
+    CHANGES = f.read()
+
+line = CHANGES.splitlines()[0]
+version = line.split()[0]
+
+setup(
+    name='maintenance',
+    version=version,
+    description='Backup PostgreSQL database, perbarui jam, dll',
+    long_description=README + '\n\n' + CHANGES,
+    classifiers=[
+        'Programming Language :: Python',
+    ],
+    author='Owo Sugiana',
+    author_email='sugiana@gmail.com',
+    url='https://git.opensipkd.com/sugiana/maintenance',
+    keywords='postgresql',
+    packages=find_packages(),
+    include_package_data=True,
+    zip_safe=False,
+    entry_points={
+        'console_scripts': [
+            'maintenance = maintenance:main',
+        ]
+    },
+)