Dapatkan SENT dari log Cartenz

CHANGES.txt

-2.0 2022-07-24
+2.0 2022-07-30
 --------------
 - Perbedaan utama dari versi 0.x adalah kini berangkat dari tabel aslinya dan
   bukan dari tabel ISO8583 karena sudah memperhatikan channel MANUAL (input

payment_report/bphtb_log_models.py

+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    DateTime,
+    Text,
+    JSON,
+    )
+from .log_models import Base
+
+
+class Log(Base):
+    __tablename__ = 'log_iso'
+    id = Column(Integer, nullable=False, primary_key=True)
+    created = Column(DateTime(timezone=True), nullable=False)
+    ip = Column(String(15))
+    mti = Column(String(4), nullable=False)
+    bit_002 = Column(Text)
+    bit_003 = Column(Text)
+    bit_004 = Column(Text)
+    bit_007 = Column(Text)
+    bit_011 = Column(Text)
+    bit_012 = Column(Text)
+    bit_013 = Column(Text)
+    bit_015 = Column(Text)
+    bit_018 = Column(Text)
+    bit_022 = Column(Text)
+    bit_032 = Column(Text)
+    bit_033 = Column(Text)
+    bit_035 = Column(Text)
+    bit_037 = Column(Text)
+    bit_039 = Column(Text)
+    bit_041 = Column(Text)
+    bit_042 = Column(Text)
+    bit_043 = Column(Text)
+    bit_047 = Column(Text)
+    bit_049 = Column(Text)
+    bit_057 = Column(Text)
+    bit_058 = Column(Text)
+    bit_059 = Column(Text)
+    bit_060 = Column(Text)
+    bit_061 = Column(Text)
+    bit_062 = Column(Text)
+    bit_063 = Column(Text)
+    bit_070 = Column(Text)
+    bit_102 = Column(Text)
+    bit_107 = Column(Text)
+    bit_047_data = Column(JSON)
+    bit_048_data = Column(JSON)

payment_report/scripts/data/conf_log2iso.csv

 nama,nilai,keterangan
-log file last id,0,Field log_file.id terakhir yang diproses
+pbb log file last id,0,Field log_file.id terakhir yang diproses
+bphtb log file last id,0,Field log_file.id terakhir yang diproses

payment_report/scripts/log2db.py

@@ -12,8 +12,6 @@ from watchdog.events import FileSystemEventHandler
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.exc import IntegrityError
-import transaction
-from zope.sqlalchemy import register
 from ..log_models import LogFile
 from .tools import BacaFile
 
@@ -22,7 +20,7 @@ my_registry = dict()
 
 PATTERN = r'^([\d]*)-([\d]*)-([\d]*) '\
           r'([\d]*):([\d]*):([\d]*)\.([\d]*) (.*) '\
-          r'\[RECEIVED RAW BUFFER\] : (.*)'
+          r'\[SENT RAW BUFFER\] : (.*)'
 REGEX = re.compile(PATTERN)
 
 
@@ -44,7 +42,7 @@ class Pantau(FileSystemEventHandler):
         set_log_file(event.src_path)
 
 
-def read_log(log_file, db_url):
+def read_log(log_file, db_url, tail_mode=True):
     def save_log(line):
         line = line.rstrip()
         match = REGEX.search(line)
@@ -54,10 +52,11 @@ def read_log(log_file, db_url):
         line_id = sha256(line.encode('utf-8')).hexdigest()
         row = LogFile(line_id=line_id, line=line)
         try:
-            with transaction.manager:
             db_session.add(row)
+            db_session.flush()
+            db_session.commit()
         except IntegrityError:
-            return
+            db_session.rollback()
 
     engine = create_engine(db_url)
     factory = sessionmaker(bind=engine)
@@ -67,7 +66,11 @@ def read_log(log_file, db_url):
             line = f.readline()
             if not line:
                 break
+            try:
                 save_log(line.decode('utf-8'))
+            except UnicodeDecodeError:
+                continue
+    if tail_mode:
         with open(log_file) as f:
             for line in tailer.follow(f):
                 save_log(line)
@@ -76,10 +79,14 @@ def read_log(log_file, db_url):
 def main(argv=sys.argv[1:]):
     pars = ArgumentParser()
     pars.add_argument('conf')
+    pars.add_argument('--log-file')
     option = pars.parse_args(argv)
     conf = ConfigParser()
     conf.read(option.conf)
     db_url = conf.get('main', 'db_url')
+    if option.log_file:
+        read_log(option.log_file, db_url, False)
+        return
     log_dir = conf.get('main', 'log_dir')
     handler = Pantau()
     observer = Observer()

payment_report/scripts/log2iso.py

@@ -20,7 +20,7 @@ from .common import (
 
 
 PATTERN = r'^([\d]*)-([\d]*)-([\d]*) ([\d]*):([\d]*):([\d]*)\.([\d]*) '\
-          r'(.*) \[RECEIVED RAW BUFFER\] : (.*)'
+          r'(.*) \[SENT RAW BUFFER\] : (.*)'
 REGEX = re.compile(PATTERN)
 
 Base = declarative_base()
@@ -40,8 +40,6 @@ def get_option(argv):
 
 
 class App(BaseApp):
-    conf_name = 'log file last id'
-
     def __init__(self, argv):
         super().__init__(argv)
         if not self.pid:
@@ -51,6 +49,10 @@ class App(BaseApp):
             pass
 
         self.report_orm = Log
+        if self.conf['models'].find('bphtb') > -1:
+            self.conf_name = 'bphtb log file last id'
+        else:
+            self.conf_name = 'pbb log file last id'
         register(self.prod_session)
 
     def get_option(self, argv):  # Override