log2db_ciamis.py
2.23 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
import sys
import os
import re
from argparse import ArgumentParser
from configparser import ConfigParser
from glob import glob
from hashlib import sha256
from time import sleep
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import (
IntegrityError,
OperationalError,
)
from ..log_models import LogFile
from ..tools import BacaFile
RE_CONNECTED = re.compile('connected$')
def read_log(log_file, db_url):
def save_log(line):
line = line.rstrip().rstrip('\x03')
print([line])
found = RE_CONNECTED.search(line)
if found:
return
print([line])
line_id = sha256(line.encode('utf-8')).hexdigest()
while True:
row = LogFile(line_id=line_id, line=line, filename=short_filename)
try:
db_session.add(row)
db_session.flush()
db_session.commit()
return
except IntegrityError:
db_session.rollback()
return
except OperationalError as e:
db_session.rollback()
print(e)
print('Tunggu, mungkin sedang maintenance ...')
sleep(10)
short_filename = os.path.split(log_file)[-1]
engine = create_engine(db_url)
factory = sessionmaker(bind=engine)
db_session = factory()
with BacaFile(log_file) as f:
while True:
line = f.readline()
if not line:
break
try:
save_log(line.decode('utf-8'))
except UnicodeDecodeError:
continue
def main(argv=sys.argv[1:]):
pars = ArgumentParser()
pars.add_argument('conf')
pars.add_argument('--log-file')
option = pars.parse_args(argv)
conf = ConfigParser()
conf.read(option.conf)
db_url = conf.get('main', 'db_url')
if option.log_file:
files = [option.log_file]
else:
dir_name = conf.get('main', 'log_dir')
pola = os.path.join(dir_name, 'sent_*_log.txt')
files = glob(pola)
files.sort()
for log_file in files:
bak_file = log_file[:-3] + 'bak'
read_log(log_file, db_url)
os.rename(log_file, bak_file)