log2db_fortuna.py
2.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
import sys
import os
import re
from argparse import ArgumentParser
from configparser import ConfigParser
from glob import glob
from hashlib import sha256
from time import sleep
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import (
IntegrityError,
OperationalError,
)
from ..log_models import LogFile
from ..tools import BacaFile
from ..common import make_pid_file
from ..common_fortuna import RE_ISO
def read_log(log_file, db_url):
def save_log(line):
line = line.rstrip()
found = RE_ISO.search(line)
if not found:
return
print([line])
line_id = sha256(line.encode('utf-8')).hexdigest()
while True:
row = LogFile(line_id=line_id, line=line)
try:
db_session.add(row)
db_session.flush()
db_session.commit()
return
except IntegrityError:
db_session.rollback()
return
except OperationalError as e:
db_session.rollback()
print(e)
print('Tunggu, mungkin sedang maintenance ...')
sleep(10)
engine = create_engine(db_url)
factory = sessionmaker(bind=engine)
db_session = factory()
with BacaFile(log_file) as f:
while True:
line = f.readline()
if not line:
break
try:
save_log(line.decode('utf-8'))
except UnicodeDecodeError:
continue
def main(argv=sys.argv[1:]):
pars = ArgumentParser()
pars.add_argument('conf')
pars.add_argument('--log-file')
option = pars.parse_args(argv)
conf = ConfigParser()
conf.read(option.conf)
make_pid_file(conf.get('main', 'pid_file'))
db_url = conf.get('main', 'db_url')
if option.log_file:
files = [option.log_file]
else:
dir_name = conf.get('main', 'log_dir')
pola = os.path.join(dir_name, 'FSwitch.BJB.*.log')
files = glob(pola)
for log_file in files:
print(log_file)
bak_file = log_file[:-3] + 'bak'
read_log(log_file, db_url)
os.rename(log_file, bak_file)