log2db_ciamis.py

import sys
import os
import re
from argparse import ArgumentParser
from configparser import ConfigParser
from hashlib import sha256
from time import sleep
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import (
    IntegrityError,
    OperationalError,
    )
from ..log_models import LogFile
from .tools import BacaFile


RE_CONNECTED = re.compile('connected$')


def read_log(log_file, db_url):
    def save_log(line):
        line = line.rstrip().rstrip('\x03')
        print([line])
        found = RE_CONNECTED.search(line)
        if found:
            return
        print([line])
        line_id = sha256(line.encode('utf-8')).hexdigest()
        while True:
            row = LogFile(line_id=line_id, line=line)
            try:
                db_session.add(row)
                db_session.flush()
                db_session.commit()
                return
            except IntegrityError:
                db_session.rollback()
                return
            except OperationalError as e:
                db_session.rollback()
                print(e)
                print('Tunggu, mungkin sedang maintenance ...')
                sleep(10)

    engine = create_engine(db_url)
    factory = sessionmaker(bind=engine)
    db_session = factory()
    with BacaFile(log_file) as f:
        while True:
            line = f.readline()
            if not line:
                break
            try:
                save_log(line.decode('utf-8'))
            except UnicodeDecodeError:
                continue


def main(argv=sys.argv[1:]):
    pars = ArgumentParser()
    pars.add_argument('conf')
    pars.add_argument('--log-file', required=True)
    option = pars.parse_args(argv)
    conf = ConfigParser()
    conf.read(option.conf)
    db_url = conf.get('main', 'db_url')
    read_log(option.log_file, db_url)