Tambah By ID selain By Date

CHANGES.txt

+3.1 2023-06-28
+--------------
+- Tambah bin/payment_report_by_id
+
 3.0.2 2023-05-17
 ----------------
 - PBB kini memahami tabel log_iso dengan struktur universal (JSON).

payment_report/common.py

@@ -9,8 +9,6 @@ from datetime import (
     datetime,
     timedelta,
     )
-from argparse import ArgumentParser
-from configparser import ConfigParser
 from ISO8583.ISO8583 import BitNotSet
 from sqlalchemy import (
     Column,
@@ -214,6 +212,9 @@ class Conf(BaseConf):
             pola += '.%f'
         return datetime.strptime(self.nilai, pola)
 
+    def as_int(self):
+        return int(self.nilai)
+
 
 def create_session(db_url, debug=False):
     engine = create_engine(db_url, echo=debug)
@@ -299,11 +300,11 @@ class BaseApp:
     def get_session_for_save(self):
         return self.rpt_session
 
-    def update_from_date(self):
+    def do_sync(self):
         q = self.get_payment_query()
         no = self.offset
         found = False
-        log = getLogger('update_from_date()')
+        log = getLogger('do_sync()')
         for pay in q.offset(self.offset).limit(row_limit):
             found = True
             no += 1
@@ -351,13 +352,13 @@ class BaseApp:
         self.last.nilai = self.get_last_time()
         with transaction.manager:
             self.rpt_session.add(self.last)
+            self.rpt_session.flush()
+            self.rpt_session.expunge_all()
 
     def get_count(self):  # Override, please
         pass
 
-    def run(self):
-        self.last_pay = self.last = None
-        self.offset = 0
+    def prepare_query_filter(self):
         if 'tgl_awal' in self.conf:
             self.tgl_awal = self.conf['tgl_awal']
         else:
@@ -367,12 +368,17 @@ class BaseApp:
             self.tgl_akhir = self.conf['tgl_akhir']
         else:
             self.tgl_akhir = date.today()
+
+    def run(self):
+        self.last_pay = self.last = None
+        self.offset = 0
+        self.prepare_query_filter()
         while True:
             self.count = self.get_count()
             if not self.count:
                 return
             self.start_time = time()
-            found = self.update_from_date()
+            found = self.do_sync()
             if not found:
                 break
             if self.last_pay and self.last:
@@ -404,3 +410,15 @@ class BaseApp:
         raise Exception(
                 f'Invoice ID {invoice_id} va_type {inv.va_type} '
                 'belum dipahami, perbaiki script')
+
+
+class BaseAppById(BaseApp):
+    def prepare_query_filter(self):  # Override
+        if self.conf.get('start_id'):
+            self.last_id = self.conf.get('start_id') - 1
+        else:
+            self.last = self.get_last_id(self.conf_name)
+            self.last_id = self.last.as_int()
+
+    def get_last_time(self):  # Override
+        return str(self.last_pay.id)

payment_report/scripts/log2iso_ciamis.py → payment_report/log2iso.py

@@ -6,35 +6,19 @@ from sqlalchemy import func
 from zope.sqlalchemy import register
 from opensipkd.waktu import create_datetime
 from opensipkd.string import FixLength
-from ..models import Common
-from ..log_models import LogFile
+from .models import Common
+from .log_models import LogFile
 from .common import (
-    BaseApp,
+    BaseAppById,
     my_registry,
-    append_csv,
     create_session,
     InvalidSource,
     )
 
 
-def get_parser():
-    pars = ArgumentParser()
-    pars.add_argument('conf')
-    pars.add_argument('--update-from-id', type=int)
-    pars.add_argument('--debug-sql', action='store_true')
-    return pars
-
-
-def get_option(argv):
-    pars = get_parser()
-    return pars.parse_args(argv)
-
-
-class App(BaseApp):
-    def __init__(self, argv):
-        super().__init__(argv)
-        if not self.pid:
-            return
+class App(BaseAppById):
+    def __init__(self, conf):
+        super().__init__(conf)
 
         class Log(self.models.Log, Common):
             pass
@@ -48,9 +32,6 @@ class App(BaseApp):
             self.pajak = 'pbb'
         register(self.prod_session)
 
-    def get_option(self, argv):  # Override
-        return get_option(argv)
-
     def get_session_for_save(self):  # Override
         return self.prod_session
 
@@ -83,7 +64,12 @@ class App(BaseApp):
                 raise InvalidSource(f'ID {log_file.id} bukan payment')
         except BitNotSet:
             raise InvalidSource(f'ID {log_file.id} bit 3 tidak ada')
-        waktu = iso.get_transaction_time()
+        # filename = 'sent_220103_log.txt'
+        tahun = '20' + log_file.filename[5:7]  # 2022
+        tahun = int(tahun)
+        raw = iso.getBit(13) + iso.getBit(12)
+        iso.transaction_time.set_raw(raw)
+        waktu = iso.transaction_time.get_value(tahun)
         d = dict(
                 id=log_file.id, mti=iso.getMTI(), created=waktu,
                 bits=iso.get_values())
@@ -102,29 +88,3 @@ class App(BaseApp):
             profile.set_raw(iso.getBit(62))
             d['bits_data'] = {62: profile.to_dict()}
         return d
-
-    def get_last_time(self):  # Override
-        return str(self.last_pay.id)
-
-    def run_payment(self):  # Override
-        self.last_pay = None
-        self.offset = 0
-        if self.option.update_from_id is None:
-            self.last = self.get_last_id(self.conf_name)
-            self.last_id = self.last.as_int()
-        else:
-            self.last = None
-            self.last_id = self.option.update_from_id - 1
-        self.count = self.get_count()
-        self.start_time = time()
-        while True:
-            found = self.update_from_date()
-            if not found:
-                break
-            if self.last_pay and self.last:
-                self.update_last()
-
-
-def main(argv=sys.argv[1:]):
-    app = App(argv)
-    app.run()

payment_report/log_models.py

@@ -15,3 +15,4 @@ class LogFile(Base):
     id = Column(Integer, primary_key=True)
     line_id = Column(String(64), nullable=False, unique=True)
     line = Column(Text, nullable=False)
+    filename = Column(String(32), nullable=False)

payment_report/scripts/log2db_ciamis.py

@@ -13,7 +13,7 @@ from sqlalchemy.exc import (
     OperationalError,
     )
 from ..log_models import LogFile
-from .tools import BacaFile
+from ..tools import BacaFile
 
 
 RE_CONNECTED = re.compile('connected$')
@@ -29,7 +29,7 @@ def read_log(log_file, db_url):
         print([line])
         line_id = sha256(line.encode('utf-8')).hexdigest()
         while True:
-            row = LogFile(line_id=line_id, line=line)
+            row = LogFile(line_id=line_id, line=line, filename=short_filename)
             try:
                 db_session.add(row)
                 db_session.flush()
@@ -44,6 +44,7 @@ def read_log(log_file, db_url):
                 print('Tunggu, mungkin sedang maintenance ...')
                 sleep(10)
 
+    short_filename = os.path.split(log_file)[-1]
     engine = create_engine(db_url)
     factory = sessionmaker(bind=engine)
     db_session = factory()
@@ -72,6 +73,7 @@ def main(argv=sys.argv[1:]):
         dir_name = conf.get('main', 'log_dir')
         pola = os.path.join(dir_name, 'sent_*_log.txt')
         files = glob(pola)
+        files.sort()
     for log_file in files:
         bak_file = log_file[:-3] + 'bak'
         read_log(log_file, db_url)

payment_report/scripts/sync.py

@@ -61,8 +61,8 @@ def get_option(argv):
     return pars.parse_args(argv)
 
 
-def main(argv=sys.argv):
-    option = get_option(argv[1:])
+def main(argv=sys.argv[1:]):
+    option = get_option(argv)
     conf = ConfigParser()
     conf.read(option.conf)
     cf = dict(conf.items('main'))

payment_report/scripts/sync_id.py

+import sys
+from configparser import ConfigParser
+from argparse import ArgumentParser
+from opensipkd.iso8583.bjb.scripts.common import get_module_object
+from .logger import setup_logging
+from .sync import make_pid_file
+
+
+def get_parser():
+    pars = ArgumentParser()
+    pars.add_argument('conf')
+    pars.add_argument('--update-from-id', type=int)
+    return pars
+
+
+def get_option(argv):
+    pars = get_parser()
+    return pars.parse_args(argv)
+
+
+def main(argv=sys.argv[1:]):
+    option = get_option(argv)
+    conf = ConfigParser()
+    conf.read(option.conf)
+    cf = dict(conf.items('main'))
+    if not make_pid_file(cf['pid_file']):
+        return
+    setup_logging(option.conf)
+    if option.update_from_id:
+        cf['start_id'] = option.update_from_id
+    module = get_module_object(conf.get('main', 'module'))
+    app = module.App(cf)
+    app.run()

setup.py

@@ -36,10 +36,10 @@ setup(
         'console_scripts': [
             'payment_report_init_db = payment_report.scripts.init_db:main',
             'payment_report = payment_report.scripts.sync:main',
+            'payment_report_by_id = payment_report.scripts.sync_id:main',
             'pbb_log = payment_report.scripts.pbb_log:main',
             'log2iso_init_db = payment_report.scripts.log2iso_init:main',
             'log2iso = payment_report.scripts.log2iso:main',
-            'log2iso_ciamis = payment_report.scripts.log2iso_ciamis:main',
             'log2iso_fortuna = payment_report.scripts.log2iso_fortuna:main',
             'log2db = payment_report.scripts.log2db:main',
             'log2db_ciamis = payment_report.scripts.log2db_ciamis:main',