Tambah baris models pada file konfigurasi sebagai pengganti module

CHANGES.txt

+0.3 28-2-2022
+-------------
+- Tambah baris models pada file konfigurasi sebagai pengganti module
+- Perbaikan pada BPHTB terkait tabel bphtb_perolehan yang berbeda schema
+- Tambah BPHTB JSON (PT POS)
+- Tambah PBB JSON (PT POS)
+
 0.2.1 23-12-2021
 ----------------
 - Tambah WEBR

bphtb.ini

+[main]
+models = opensipkd.bphtb.models.default
+db_url = postgresql://user:pass@localhost/db
+report_db_url = postgresql://user:pass@localhost/db
+pid_file = /home/sugiana/tmp/bphtb-report.pid
+log_file = /home/sugiana/log/bphtb-report.log

pad.ini

+[main]
+models = opensipkd.pad.models.default
+db_url = postgresql://user:pass@localhost/db
+report_db_url = postgresql://user:pass@localhost/db
+pid_file = /home/sugiana/tmp/pad-report.pid
+log_file = /home/sugiana/log/pad-report.log

payment_report/models.py

@@ -51,6 +51,8 @@ class Pad(Base, Common):
     denda = Column(Float, nullable=False)
     # pad.pad_sspd.jml_bayar
     jml_bayar = Column(Float, nullable=False)
+    # Bit 32
+    bank_id = Column(Integer)
     # Bit 18
     channel_id = Column(String(4), nullable=False)
     # Bit 41 / 42 / 43

payment_report/scripts/bphtb.py

@@ -50,7 +50,7 @@ class IsoPayment(Base):
 
 class IsoReversal(Base):
     __tablename__ = 'bphtb_reversal'
-    __table_args__ = dict(schema='bphtb')
+    __table_args__ = dict(schema='public')
     id = Column(Integer, ForeignKey(IsoPayment.id), primary_key=True)
     tgl = Column(DateTime(timezone=True), nullable=False)
     iso_request = Column(String(1024), nullable=False)
@@ -79,6 +79,11 @@ class App(BaseApp):
                 IsoPayment.id == Payment.id,
                 IsoPayment.id > last.as_int())
         for row_iso, row_pay in q_iso.order_by(IsoPayment.id):
+            if row_iso.iso_request[0] == '{':
+                last.nilai = str(row_iso.id)
+                with transaction.manager:
+                    self.rpt_session.add(last)
+                continue
             iso = get_iso(row_iso.iso_request, Doc, self.option.debug)
             if self.get_report(iso):
                 continue
@@ -102,7 +107,6 @@ class App(BaseApp):
                     channel_id=iso.get_channel().strip(),
                     channel_nama=d['channel'])
             last.nilai = str(row_iso.id)
-            print(f'last.nilai {last.nilai}')
             with transaction.manager:
                 self.rpt_session.add(rpt)
                 self.rpt_session.add(last)

payment_report/scripts/bphtb2.py

 import sys
+from sqlalchemy.exc import ProgrammingError
 import transaction
-from opensipkd.waktu import dmyhms
+from opensipkd.waktu import (
+    dmyhms,
+    date_from_str,
+    )
+from opensipkd.bphtb.models.perolehan import PerolehanMixin
 from opensipkd.bphtb.models.default import Perolehan
-from opensipkd.bphtb.services.base import get_db_session
 from iso8583_web.models.meta import Base as BaseConf
 from opensipkd.iso8583.bjb.bphtb.structure import PAYMENT_CODE
 from opensipkd.iso8583.bjb.bphtb.models import Log
@@ -16,39 +20,42 @@ from .common import (
     )
 
 
+class AlternativePerolehan(Base, PerolehanMixin):
+    __table_args__ = dict(schema='public')
+
+
 class App(BaseApp):
     field_invoice_id = 'bit_062'
     field_ntb = 'bit_058'
     report_orm = Bphtb
-    log_orm = Log
 
     def __init__(self, argv):
         super().__init__(argv)
         if not self.pid:
             return
         self.base_q_perolehan = self.prod_session.query(Perolehan)
+        with transaction.manager:
+            try:
+                self.base_q_perolehan.first()
+            except ProgrammingError:
+                self.prod_session.rollback()
+                self.base_q_perolehan = self.prod_session.query(
+                        AlternativePerolehan)
 
-    def get_db_session(self):  # Override
-        return get_db_session()
+    def get_log_orm(self):  # Override
+        return Log
 
-    def run_payment(self):  # Override
-        last = self.get_last_id('bphtb2 payment last id')
-        q_iso = self.prod_session.query(Log).filter_by(
-                    mti='0210', bit_003=PAYMENT_CODE, bit_039='00').filter(
-                    Log.id > last.as_int())
-        for row_iso in q_iso.order_by(Log.id):
-            if self.get_report(row_iso):
-                continue
+    def __create_data(self, row_iso):
         tgl_bayar = row_iso.created.date()
         d = self.get_keys(row_iso)
         s_tgl = dmyhms(row_iso.created)
         self.log.info(
             f'Tgl bayar {s_tgl}, Nomor bayar {d["nomor_bayar"]}, '
             f'STAN {d["stan"]}, NTB {d["ntb"]}, Channel {d["channel"]}')
-            p = row_iso.bit_047_data
+        p = eval(row_iso.bit_047_data)
         q = self.base_q_perolehan.filter_by(id=p['Jenis Perolehan Hak'])
         perolehan = q.first()
-            rpt = Bphtb(
+        return dict(
             stan=d['stan'], ntb=d['ntb'], tgl=tgl_bayar,
             jam=row_iso.created.time(),
             invoice_id=row_iso.bit_062.strip(),
@@ -64,11 +71,63 @@ class App(BaseApp):
             ppat=p['Nama Notaris'],
             channel_id=row_iso.bit_018.strip(),
             channel_nama=d['channel'])
+
+    def __get_query_iso(self, last_id):
+        return self.prod_session.query(Log).filter_by(
+                mti='0210', bit_003=PAYMENT_CODE, bit_039='00').filter(
+                Log.id > last_id)
+
+    def __run_payment(self):
+        last = self.get_last_id('bphtb2 payment last id')
+        q_iso = self.__get_query_iso(last.as_int())
+        found = False
+        for row_iso in q_iso.order_by(Log.id).limit(1000):
+            if self.get_report(row_iso):
+                last.nilai = str(row_iso.id)
+                with transaction.manager:
+                    self.rpt_session.add(last)
+                continue
+            data = self.__create_data(row_iso)
+            rpt = Bphtb(**data)
             last.nilai = str(row_iso.id)
-            print(f'last.nilai {last.nilai}')
             with transaction.manager:
                 self.rpt_session.add(rpt)
                 self.rpt_session.add(last)
+            found = True
+        return found
+
+    def __update_from_id(self):
+        q_iso = self.__get_query_iso(self.last_id)
+        found = False
+        for row_iso in q_iso.order_by(Log.id).limit(1000):
+            rpt = self.get_report(row_iso)
+            if not rpt:
+                rpt = Bphtb()
+            data = self.__create_data(row_iso)
+            rpt.from_dict(data)
+            with transaction.manager:
+                self.rpt_session.add(rpt)
+            self.last_id = row_iso.id
+            found = True
+        return found
+
+    def run_payment(self):  # Override
+        if self.option.update_from_id:
+            self.last_id = self.option.update_from_id
+            func = self.__update_from_id
+        elif self.option.update_from_date:
+            tgl = date_from_str(self.option.update_from_date)
+            q = self.prod_session.query(Log).filter(Log.created >= tgl)
+            q = q.order_by(Log.id)
+            row = q.first()
+            self.last_id = row.id - 1
+            func = self.__update_from_id
+        else:
+            func = self.__run_payment
+        while True:
+            found = func()
+            if not found:
+                break
 
     def run_reversal(self):  # Override
         super().run_reversal('bphtb2 reversal last id')

payment_report/scripts/bphtb_json.py

+import sys
+from sqlalchemy import (
+    Column,
+    Integer,
+    DateTime,
+    String,
+    Date,
+    ForeignKey,
+    )
+import transaction
+import datetime
+from opensipkd.waktu import (
+    dmyhms,
+    date_from_str,
+    )
+from opensipkd.bphtb.models.default import (
+    Payment,
+    Invoice,
+    Perolehan,
+    Customer,
+    )
+from opensipkd.bphtb.services.base import get_db_session
+from iso8583_web.models.meta import Base as BaseConf
+from opensipkd.iso8583.bjb.bphtb import Doc
+from ..models import (
+    Base,
+    Bphtb,
+    )
+from .common import (
+    BaseApp,
+    init_db as base_init_db,
+    BANK_NAMES,
+    )
+
+
+def get_keys(iso):
+    return dict(
+            nomor_bayar=iso['no_tagihan'],
+            ntb=iso['transno'],
+            nama_bank=BANK_NAMES[iso['bankid']])
+
+
+def get_filter(q):
+    return q.filter(IsoPayment.iso_request.like('{%'))
+
+
+class IsoPayment(Base):
+    __tablename__ = 'bphtb_payment'
+    __table_args__ = dict(schema='bphtb')
+    id = Column(Integer, ForeignKey(Payment.id), primary_key=True)
+    tgl = Column(DateTime(timezone=True), nullable=False)
+    iso_request = Column(String(1024), nullable=False)
+    transmission = Column(DateTime(timezone=True), nullable=False)
+    settlement = Column(Date, nullable=False)
+    stan = Column(Integer, nullable=False)
+    invoice_id = Column(Integer, ForeignKey(Invoice.id), nullable=False)
+    invoice_no = Column(String(32), nullable=False)
+    ntb = Column(String(32), nullable=False)
+    ntp = Column(String(32), nullable=False, unique=True)
+    bank_id = Column(Integer)
+    channel_id = Column(Integer)
+    bank_ip = Column(String(15), nullable=False)
+
+
+class IsoReversal(Base):
+    __tablename__ = 'bphtb_reversal'
+    __table_args__ = dict(schema='public')
+    id = Column(Integer, ForeignKey(IsoPayment.id), primary_key=True)
+    tgl = Column(DateTime(timezone=True), nullable=False)
+    iso_request = Column(String(1024), nullable=False)
+
+
+class App(BaseApp):
+    iso_class = Doc
+    report_orm = Bphtb
+    iso_reversal_orm = IsoReversal
+
+    def __init__(self, argv):
+        super().__init__(argv)
+        if not self.pid:
+            return
+        self.base_q_inv = self.prod_session.query(
+                Invoice, Customer, Perolehan).filter(
+                Invoice.ppat_id == Customer.id,
+                Invoice.perolehan_id == Perolehan.id)
+
+    def get_db_session(self):  # Override
+        return get_db_session()
+
+    def update_last_id(self, last, value):
+        last.nilai = str(value)
+        with transaction.manager:
+            self.rpt_session.add(last)
+
+    def __warning_nol(self, s_tgl, d, row_pay):
+        msg = f'Tgl bayar {s_tgl}, Nomor bayar {d["nomor_bayar"]}, '\
+                f'NTB {d["ntb"]}, Channel {d["nama_bank"]}, '\
+                f'nilai Rp {row_pay.bayar} tidak disimpan'
+        self.log.warning(msg)
+
+    def __log_progress(self, s_tgl, d):
+        self.log.info(
+            f'Tgl bayar {s_tgl}, Nomor bayar {d["nomor_bayar"]}, '
+            f'NTB {d["ntb"]}, Channel {d["nama_bank"]}')
+
+    def __get_query_iso(self, last_id):
+        q_iso = self.prod_session.query(IsoPayment, Payment).filter(
+                IsoPayment.id == Payment.id,
+                IsoPayment.id > last_id)
+        q_iso = get_filter(q_iso)
+        return q_iso.order_by(IsoPayment.id)
+
+    def __create_data(self, row_iso, row_pay, iso, d):
+        q_inv = self.base_q_inv.filter(Invoice.id == row_pay.sspd_id)
+        inv, ppat, perolehan = q_inv.first()
+        return dict(
+            stan=row_iso.id, ntb=d['ntb'], tgl=row_iso.tgl,
+            jam=row_iso.tgl.time(), invoice_id=d['nomor_bayar'],
+            nop=row_pay.nop, wp_nama=row_pay.wp_nama,
+            wp_alamat=row_pay.wp_alamat, op_alamat=inv.op_alamat,
+            npop=row_pay.npop, bumi_luas=row_pay.bumi_luas,
+            bng_luas=row_pay.bng_luas, nilai_bphtb=row_pay.bayar,
+            jenis_perolehan=perolehan.nama, ppat=ppat.nama,
+            channel_id=int(iso['bankid']), channel_nama=d['nama_bank'])
+
+    def __run_payment(self):
+        last = self.get_last_id('bphtb json payment last id')
+        q_iso = self.__get_query_iso(last.as_int())
+        for row_iso, row_pay in q_iso:
+            iso = eval(row_iso.iso_request)
+            d = get_keys(iso)
+            if self.get_report(d['nomor_bayar'], d['ntb']):
+                continue
+            s_tgl = dmyhms(row_iso.tgl)
+            if not row_pay.bayar:
+                self.__warning_nol(s_tgl, d, row_pay)
+                self.update_last_id(last, str(row_iso.id))
+                continue
+            self.__log_progress(s_tgl, d)
+            data = self._create_data(row_iso, row_pay, iso, d)
+            rpt = Bphtb(**data)
+            last.nilai = str(row_iso.id)
+            with transaction.manager:
+                self.rpt_session.add(rpt)
+                self.rpt_session.add(last)
+
+    def __update_from_id(self):
+        q_iso = self.__get_query_iso(self.last_id)
+        found = False
+        for row_iso, row_pay in q_iso.limit(1000):
+            iso = eval(row_iso.iso_request)
+            d = get_keys(iso)
+            s_tgl = dmyhms(row_iso.tgl)
+            if not row_pay.bayar:
+                self.__warning_nol(s_tgl, d, row_pay)
+                continue
+            self.__log_progress(s_tgl, d)
+            data = self.__create_data(row_iso, row_pay, iso, d)
+            rpt = self.get_report(d['nomor_bayar'], d['ntb'])
+            if not rpt:
+                rpt = Bphtb()
+            rpt.from_dict(data)
+            with transaction.manager:
+                self.rpt_session.add(rpt)
+            found = True
+            self.last_id = row_iso.id
+        return found
+
+    def run_payment(self):  # Override
+        if self.option.update_from_id:
+            self.last_id = self.option.update_from_id
+            func = self.__update_from_id
+        elif self.option.update_from_date:
+            tgl = date_from_str(self.option.update_from_date)
+            q = self.prod_session.query(IsoPayment).filter(
+                    IsoPayment.tgl >= tgl).order_by(
+                    IsoPayment.id)
+            q = get_filter(q)
+            row = q.first()
+            self.last_id = row.id - 1
+            func = self.__update_from_id
+        else:
+            func = self.__run_payment
+        while True:
+            found = func()
+            if not found:
+                break
+
+    def run_reversal(self):  # Override
+        last = self.get_last_id('bphtb json reversal last date')
+        q = self.prod_session.query(self.iso_reversal_orm).filter(
+                self.iso_reversal_orm.iso_request.like('{%'),
+                self.iso_reversal_orm.tgl > last.as_datetime())
+        for row in q.order_by(self.iso_reversal_orm.tgl):
+            iso = eval(row.iso_request)
+            invoice_id = iso['bit_061']
+            ntb = iso['bit_037']
+            rpt = self.get_report(invoice_id, ntb)
+            if not rpt or rpt.tgl_batal:
+                continue
+            nama_bank = BANK_NAMES[iso['bit_032']]
+            rpt.tgl_batal = row.tgl
+            last.nilai = s_tgl = dmyhms(row.tgl)
+            self.log.info(
+                f'Tgl batal {s_tgl}, Nomor bayar {invoice_id}, '
+                f'NTB {ntb}, Channel {nama_bank}')
+            with transaction.manager:
+                self.rpt_session.add(rpt)
+                self.rpt_session.add(last)
+
+
+def main(argv=sys.argv[1:]):
+    app = App(argv)
+    if app.pid:
+        app.run()
+
+
+def init_db(argv=sys.argv[1:]):
+    base_init_db(Base.metadata, BaseConf.metadata, argv)

payment_report/scripts/common.py

@@ -43,7 +43,13 @@ BIT_PROFILE_NAMES = {
     'TOKOPEDI': 'TOKOPEDIA',
     'BUKALAPA': 'BUKALAPAK',
     'MASAGO': 'MASAGO',
-    'BAYARIN': 'BAYARIN'}
+    'BAYARIN': 'BAYARIN',
+    'TRAVELOK': 'TRAVELOKA',
+    'SHOPEE': 'SHOPEE',
+    'OVO': 'OVO',
+    'BLIBLI': 'BLIBLI'}
+
+BANK_NAMES = {'700': 'PT POS'}
 
 my_registry = dict()
 
@@ -90,7 +96,9 @@ def get_iso(raw, iso_class, debug=False):
     return iso
 
 
-def get_channel_name(bit_018, bit_041, bit_042, bit_043):
+def get_channel_name(bit_018, bit_032, bit_041, bit_042, bit_043):
+    if bit_032 and bit_032 in BANK_NAMES:
+        return BANK_NAMES[bit_032]
     if not bit_018:
         return 'LAINNYA'
     bit_018 = bit_018.strip()
@@ -120,14 +128,22 @@ def get_channel_info_by_iso(iso):
             continue
     if lengkap:
         d['channel'] = get_channel_name(
-                iso.getBit(18), iso.getBit(41), iso.getBit(42), iso.getBit(43))
+                iso.getBit(18), iso.getBit(32), iso.getBit(41), iso.getBit(42),
+                iso.getBit(43))
     else:
         d['channel'] = 'LAINNYA'
     return d
 
 
 def get_channel_name_by_row(row):
-    return get_channel_name(row.bit_018, row.bit_041, row.bit_042, row.bit_043)
+    return get_channel_name(
+            row.bit_018, row.bit_032, row.bit_041, row.bit_042, row.bit_043)
+
+
+def get_channel_name_by_dict(d):
+    return get_channel_name(
+            d['bit_018'], d['bit_032'], d.get('bit_041'), d.get('bit_042'),
+            d.get('bit_043'))
 
 
 def get_keys(iso):
@@ -223,21 +239,20 @@ class BaseApp:
         if not self.pid:
             return
         self.log = create_log(self.conf['log_file'])
-        module_name = self.conf['module']
-        self.module = get_module_object(module_name)
+        if 'models' in self.conf:
+            self.models = get_module_object(self.conf['models'])
         module_conf = dict(self.conf)
-        self.module.init(module_conf)
-        self.prod_session = self.get_db_session()
-        self.prod_session.bind.echo = self.option.debug_sql
-        engine = create_engine(
-                self.conf['report_db_url'], echo=self.option.debug_sql)
-        factory = sessionmaker(bind=engine)
+        factory = self.get_factory('db_url')
+        self.prod_session = factory()
+        factory = self.get_factory('report_db_url')
         self.rpt_session = factory()
         register(self.rpt_session)
         self.base_q_report = self.rpt_session.query(self.report_orm)
 
-    def get_db_session(self):  # Override, please
-        pass
+    def get_factory(self, name):
+        db_url = self.conf[name]
+        engine = create_engine(db_url, echo=self.option.debug_sql)
+        return sessionmaker(bind=engine)
 
     def get_last_id(self, nama):
         q = self.rpt_session.query(Conf).filter_by(nama=nama)
@@ -263,20 +278,28 @@ class BaseApp:
 # Tanpa tabel log_iso
 
 class App(BaseApp):
-    iso_class = None  # Override, please
-    iso_reversal_orm = None  # Override, please
-
     def get_report(self, iso):  # Override
         d = get_keys(iso)
         return super().get_report(d['stan'], d['ntb'])
 
+    def get_iso_reversal_orm(self):  # Override, please
+        return
+
+    def get_doc_for_reversal(row):  # Override, please
+        return
+
     def run_reversal(self, conf_name):  # Override
         last = self.get_last_id(conf_name)
-        q = self.prod_session.query(self.iso_reversal_orm).filter(
-                self.iso_reversal_orm.tgl > last.as_datetime())
-        iso_class = self.module.Doc()
-        for row in q.order_by(self.iso_reversal_orm.tgl):
-            iso = get_iso(row.iso_request, self.iso_class, self.option.debug)
+        orm = self.get_iso_reversal_orm()
+        q = self.prod_session.query(orm).filter(orm.tgl > last.as_datetime())
+        for row in q.order_by(orm.tgl):
+            if row.iso_request[0] == '{':
+                last.nilai = dmyhms(row.tgl)
+                with transaction.manager:
+                    self.rpt_session.add(last)
+                continue
+            Doc = self.get_doc_for_reversal(row)
+            iso = get_iso(row.iso_request, Doc, self.option.debug)
             rpt = self.get_report(iso)
             if not rpt or rpt.tgl_batal:
                 continue
@@ -294,10 +317,12 @@ class App(BaseApp):
 # Dengan tabel log_iso
 
 class App2(BaseApp):
-    log_orm = None  # Override, please
     field_invoice_id = None  # Override, please
     field_ntb = None  # Override, please
 
+    def get_log_orm(self):  # Override, please
+        return
+
     def get_stan(self, row):
         return row.bit_011.strip()
 
@@ -323,10 +348,11 @@ class App2(BaseApp):
 
     def run_reversal(self, conf_name):  # Override
         last = self.get_last_id(conf_name)
-        q = self.prod_session.query(self.log_orm).filter_by(
+        orm = self.get_log_orm()
+        q = self.prod_session.query(orm).filter_by(
                 mti='0410', bit_039='00').filter(
-                self.log_orm.id > last.id)
-        for row in q.order_by(self.log_orm.id):
+                orm.id > last.id)
+        for row in q.order_by(orm.id):
             rpt = self.get_report(row)
             if not rpt or rpt.tgl_batal:
                 continue

payment_report/scripts/data/conf.csv

@@ -7,6 +7,8 @@ webr payment last id,0,log_iso.id terakhir yang diproses
 webr reversal last id,0,log_iso.id terakhir yang diproses
 bphtb payment last id,0,bphtb.bphtb_payment.id terakhir yang diproses
 bphtb reversal last date,1-1-2000 00:00:00,bphtb.bphtb_reversal.tgl terakhir yang diproses
+bphtb json payment last id,0,bphtb.bphtb_payment.id terakhir yang diproses
+bphtb json reversal last date,1-1-2000 00:00:00,bphtb.bphtb_reversal.tgl terakhir yang diproses
 bphtb2 payment last id,0,log_iso.id terakhir yang diproses
 bphtb2 reversal last id,0,log_iso.id terakhir yang diproses
 pbb payment last id,0,inquiry.id terakhir yang diproses

payment_report/scripts/pad.py

 import sys
 from datetime import timedelta
 import transaction
-from opensipkd.waktu import dmyhms
+from opensipkd.waktu import (
+    dmyhms,
+    date_from_str,
+    )
 from opensipkd.pad.models.default import (
     IsoPayment,
     IsoReversal,
-    Payment,
-    Invoice,
-    Pajak,
-    CustomerUsaha,
-    Customer,
-    Usaha,
     )
-from opensipkd.pad.services.base import get_db_session
 from iso8583_web.models.meta import Base as BaseConf
-from opensipkd.iso8583.bjb.pad import Doc
+from opensipkd.iso8583.bjb.pad.doc import Doc as BjbDoc
+from opensipkd.iso8583.multi.doc import Doc as MultiDoc
 from ..models import (
     Base,
     Pad,
@@ -26,19 +23,27 @@ from .common import (
     init_db as base_init_db,
     )
 
+BankHandlers = {
+    8: MultiDoc,
+    14: MultiDoc,
+    110: BjbDoc}
 
 ERR_NOT_FOUND = 'Tgl {tgl_bayar} pad.pad_sspd.spt_id {invoice_id} tidak ada'
 
 
 class App(BaseApp):
-    iso_class = Doc
     report_orm = Pad
-    iso_reversal_orm = IsoReversal
 
     def __init__(self, argv):
         super().__init__(argv)
         if not self.pid:
             return
+        Usaha = self.models.Usaha
+        Pajak = self.models.Pajak
+        Customer = self.models.Customer
+        CustomerUsaha = self.models.CustomerUsaha
+        Invoice = self.models.Invoice
+        Payment = self.models.Payment
         self.base_q_pay = self.prod_session.query(Payment)
         self.base_q_inv = self.prod_session.query(
                 Invoice, Pajak, Customer, Usaha)
@@ -47,23 +52,45 @@ class App(BaseApp):
                 Invoice.customer_usaha_id == CustomerUsaha.id,
                 CustomerUsaha.customer_id == Customer.id,
                 Usaha.id == Pajak.usaha_id,)
-
-    def get_db_session(self):  # Override
-        return get_db_session()
+        self.base_q_iso = self.prod_session.query(IsoPayment)
 
     def get_pay(self, row):
         tgl_bayar = row.tgl.date()
+        Payment = self.models.Payment
         q_pay = self.base_q_pay.filter_by(spt_id=row.invoice_id).filter(
                 Payment.create_date >= tgl_bayar,
                 Payment.create_date < tgl_bayar + timedelta(1))
         q_pay = q_pay.order_by(Payment.id.desc())
         return q_pay.first()
 
-    def run_payment(self):
+    def __get_query_iso(self, last_id):
+        q_iso = self.base_q_iso.filter(IsoPayment.id > last_id)
+        return q_iso.order_by(IsoPayment.id)
+
+    def __create_data(self, row, pay, iso, d, tgl_bayar):
+        Invoice = self.models.Invoice
+        q_inv = self.base_q_inv.filter(Invoice.id == pay.spt_id)
+        inv, pajak, cust, usaha = q_inv.first()
+        return dict(
+            stan=d['stan'], ntb=d['ntb'], tgl=tgl_bayar,
+            jam=row.tgl.time(), sspd_id=pay.id,
+            nomor_bayar=iso.get_invoice_id().strip(),
+            jenis_pajak=usaha.usahanm.strip(),
+            masa_pajak=pajak.masapajak, npwpd=cust.npwpd,
+            nama_wp=cust.customernm, pokok=pay.jml_bayar-pay.denda,
+            denda=pay.denda, jml_bayar=pay.jml_bayar,
+            channel_id=iso.get_channel().strip(),
+            channel_name=d['channel'], bank_id=iso.get_bank_id())
+
+    def __log_progress(self, s_tgl, d):
+        self.log.info(f'Tgl bayar {s_tgl}, {d}')
+
+    def __run_payment(self):
         last = self.get_last_id('pad payment last id')
-        q_iso = self.prod_session.query(IsoPayment).filter(
-                IsoPayment.id > last.as_int())
-        for row in q_iso.order_by(IsoPayment.id):
+        q_iso = self.__get_query_iso(last.as_int())
+        found = False
+        for row in q_iso.order_by(IsoPayment.id).limit(1000):
+            Doc = BankHandlers[row.bank_id]
             iso = get_iso(row.iso_request, Doc, self.option.debug)
             if self.get_report(iso):
                 continue
@@ -74,28 +101,71 @@ class App(BaseApp):
                         tgl_bayar=tgl_bayar, invoice_id=row.invoice_id)
                 self.log.error(msg)
                 continue
-            q_inv = self.base_q_inv.filter(Invoice.id == pay.spt_id)
-            inv, pajak, cust, usaha = q_inv.first()
             d = get_keys(iso)
             s_tgl = dmyhms(row.tgl)
-            self.log.info(
-                f'Tgl bayar {s_tgl}, Nomor bayar {d["nomor_bayar"]}, '
-                f'Jenis Pajak {usaha.usahanm.strip()},'
-                f'STAN {d["stan"]}, NTB {d["ntb"]}, Channel {d["channel"]}')
-            rpt = Pad(
-                    stan=d['stan'], ntb=d['ntb'], tgl=tgl_bayar,
-                    jam=row.tgl.time(), sspd_id=pay.id,
-                    nomor_bayar=iso.get_invoice_id().strip(),
-                    jenis_pajak=usaha.usahanm.strip(),
-                    masa_pajak=pajak.masapajak, npwpd=cust.npwpd,
-                    nama_wp=cust.customernm, pokok=pay.jml_bayar-pay.denda,
-                    denda=pay.denda, jml_bayar=pay.jml_bayar,
-                    channel_id=iso.get_channel().strip(),
-                    channel_name=d['channel'])
+            data = self.__create_data(row, pay, iso, d, tgl_bayar)
+            self.__log_progress(s_tgl, data)
+            rpt = Pad(**data)
             last.nilai = str(row.id)
             with transaction.manager:
                 self.rpt_session.add(rpt)
                 self.rpt_session.add(last)
+            found = True
+        return found
+
+    def __update_from_id(self):
+        q_iso = self.__get_query_iso(self.last_id)
+        found = False
+        for row in q_iso.order_by(IsoPayment.id).limit(1000):
+            Doc = BankHandlers[row.bank_id]
+            iso = get_iso(row.iso_request, Doc, self.option.debug)
+            tgl_bayar = row.tgl.date()
+            pay = self.get_pay(row)
+            rpt = self.get_report(iso)
+            if not rpt:
+                if not pay:
+                    msg = ERR_NOT_FOUND.format(
+                            tgl_bayar=tgl_bayar, invoice_id=row.invoice_id)
+                    self.log.error(msg)
+                    continue
+                rpt = Pad()
+            d = get_keys(iso)
+            s_tgl = dmyhms(row.tgl)
+            data = self.__create_data(row, pay, iso, d, tgl_bayar)
+            self.__log_progress(s_tgl, data)
+            rpt.from_dict(data)
+            with transaction.manager:
+                self.rpt_session.add(rpt)
+            found = True
+            self.last_id = row.id
+        return found
+
+    def run_payment(self):  # Override
+        if self.option.update_from_id:
+            self.last_id = self.option.update_from_id
+            func = self.__update_from_id
+        elif self.option.update_from_date:
+            tgl = date_from_str(self.option.update_from_date)
+            q = self.prod_session.query(IsoPayment).filter(
+                    IsoPayment.tgl >= tgl).order_by(
+                    IsoPayment.id)
+            row = q.first()
+            self.last_id = row.id - 1
+            func = self.__update_from_id
+        else:
+            func = self.__run_payment
+        while True:
+            found = func()
+            if not found:
+                break
+
+    def get_iso_reversal_orm(self):  # Override
+        return IsoReversal
+
+    def get_doc_for_reversal(self, row):  # Override
+        q = self.prod_session.query(IsoPayment).filter_by(id=row.id)
+        iso_pay = q.first()
+        return BankHandlers[iso_pay.bank_id]
 
     def run_reversal(self):  # Override
         super().run_reversal('pad reversal last date')

payment_report/scripts/pbb.py

@@ -10,9 +10,8 @@ from opensipkd.waktu import (
     create_datetime,
     date_from_str,
     )
-from sismiop.services.base import get_db_session
 from opensipkd.iso8583.bjb.scripts.common import get_module_object
-from opensipkd.iso8583.bjb.pbb import Doc
+from opensipkd.iso8583.bjb.pbb.default import Doc
 from opensipkd.iso8583.bjb.pbb.structure import INVOICE_PROFILE
 from ..models import Pbb
 from .common import (
@@ -22,8 +21,10 @@ from .common import (
     )
 
 
-ERR_NOT_FOUND = 'Tgl {tgl_bayar} SPPT ID {invoice_id} '\
+ERR_PSPPT_NOT_FOUND = 'Tgl {tgl_bayar} SPPT ID {invoice_id} '\
         'tidak ada di pembayaran_sppt'
+ERR_SPPT_NOT_FOUND = 'Tgl {tgl_bayar} SPPT ID {invoice_id}'\
+        'tidak ada di tabel sppt'
 
 
 def get_profile(iso):
@@ -66,7 +67,7 @@ class App(BaseApp):
         super().__init__(argv)
         if not self.pid:
             return
-        d = self.module.Doc()
+        d = Doc()
         self.iso_inquiry_orm = d.iso_inquiry_model
         self.iso_payment_orm = d.iso_payment_model
         self.iso_reversal_orm = d.iso_reversal_model
@@ -79,8 +80,8 @@ class App(BaseApp):
         self.base_q_psppt = self.prod_session.query(self.PembayaranSppt)
         self.base_q_sppt = self.prod_session.query(self.Sppt)
 
-    def get_db_session(self):  # Override
-        return get_db_session()
+    def get_iso_reversal_orm(self):  # Override
+        return self.iso_reversal_orm
 
     def get_psppt(self, pay):
         q = self.base_q_psppt.filter_by(
@@ -111,8 +112,13 @@ class App(BaseApp):
         q_iso = self.base_q_iso.filter(self.iso_inquiry_orm.id > last_id)
         return q_iso.order_by(self.iso_inquiry_orm.id)
 
-    def __log_not_found(self, s_tgl, d):
-        msg = ERR_NOT_FOUND.format(
+    def __log_psppt_not_found(self, s_tgl, d):
+        msg = ERR_PSPPT_NOT_FOUND.format(
+                tgl_bayar=s_tgl, invoice_id=d['nomor_bayar'])
+        self.log.error(msg)
+
+    def __log_sppt_not_found(self, s_tgl, d):
+        msg = ERR_SPPT_NOT_FOUND.format(
                 tgl_bayar=s_tgl, invoice_id=d['nomor_bayar'])
         self.log.error(msg)
 
@@ -121,6 +127,9 @@ class App(BaseApp):
 
     def __create_data(self, row_inq, iso, d, tgl_bayar, psppt):
         sppt = self.get_sppt(psppt)
+        if not sppt:
+            self.__log_sppt_not_found(tgl_bayar, d)
+            return
         nama_wp = get_nama_wp(iso, sppt)
         user_id = get_user_id(iso)
         discount = hasattr(psppt, 'discount') and psppt.discount or None
@@ -152,10 +161,11 @@ class App(BaseApp):
             s_tgl = dmyhms(tgl_bayar)
             psppt = self.get_psppt(row_pay)
             if not psppt:
-                self.__log_not_found(s_tgl, d)
+                self.__log_psppt_not_found(s_tgl, d)
                 continue
             self.__log_progress(s_tgl, d)
             d = self.__create_data(row_inq, iso, d, tgl_bayar, psppt)
+            if d:
                 rpt = Pbb(**d)
                 last.nilai = str(row_inq.id)
                 with transaction.manager:
@@ -174,10 +184,11 @@ class App(BaseApp):
             s_tgl = dmyhms(tgl_bayar)
             psppt = self.get_psppt(row_pay)
             if not psppt:
-                self.__log_not_found(s_tgl, d)
+                self.__log_psppt_not_found(s_tgl, d)
                 continue
             self.__log_progress(s_tgl, d)
             d = self.__create_data(row_inq, iso, d, tgl_bayar, psppt)
+            if d:
                 rpt = self.get_report(iso)
                 if not rpt:
                     rpt = Pbb()
@@ -207,6 +218,9 @@ class App(BaseApp):
             if not found:
                 break
 
+    def get_doc_for_reversal(self, row):
+        return Doc
+
     def run_reversal(self):  # Override
         super().run_reversal('pbb reversal last date')
 

payment_report/scripts/pbb_json.py

+import sys
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    UniqueConstraint,
+    )
+import transaction
+from opensipkd.string import (
+    FixLength,
+    DateTimeVar,
+    )
+from opensipkd.waktu import (
+    dmyhms,
+    create_datetime,
+    date_from_str,
+    )
+from sismiop.services.base import get_id
+from ..models import (
+    Pbb as Report,
+    Base,
+    )
+from .common import (
+    BaseApp,
+    get_channel_name_by_dict,
+    get_module_object,
+    )
+
+
+ERR_NOT_FOUND = 'SPPT ID {invoice_id} pembayaran ke {ke} '\
+        'tidak ada di pembayaran_sppt'
+
+
+class Log(Base):
+    __tablename__ = 'pbb_payment'
+    id = Column(Integer, primary_key=True)
+    propinsi = Column(String(2), nullable=False)
+    kabupaten = Column(String(2), nullable=False)
+    kecamatan = Column(String(3), nullable=False)
+    kelurahan = Column(String(3), nullable=False)
+    blok = Column(String(3), nullable=False)
+    urut = Column(String(4), nullable=False)
+    jenis = Column(String(1), nullable=False)
+    tahun = Column(Integer, nullable=False)
+    ke = Column(Integer, nullable=False)
+    kd_kanwil_bank = Column(String(2), nullable=False)
+    kd_kppbb_bank = Column(String(2), nullable=False)
+    kd_bank_tunggal = Column(String(2), nullable=False)
+    kd_bank_persepsi = Column(String(2), nullable=False)
+    kd_tp = Column(String(2), nullable=False)
+    channel = Column(String(4))
+    ntb = Column(String(64))
+    ntp = Column(String(64))
+    bank = Column(Integer)
+    iso_request = Column(String(2048), nullable=False)
+    __table_args__ = (
+        UniqueConstraint('propinsi', 'kabupaten', 'kecamatan', 'kelurahan',
+                         'blok', 'urut', 'jenis', 'tahun', 'ke'),
+        dict(schema='public'))
+
+
+def get_keys(d):
+    r = dict()
+    r['channel'] = get_channel_name_by_dict(d)
+    r.update(dict(
+        nomor_bayar=d['bit_061'],
+        stan=d['bit_011'],
+        ntb=d['bit_037']))
+    return r
+
+
+def get_nama_wp(d):
+    return d['bit_062']['name']
+
+
+def get_tgl_bayar(d, year):
+    raw = d['bit_007']
+    t = DateTimeVar()
+    t.set_raw(raw)
+    return create_datetime(
+            year, int(t['month']), int(t['day']), int(t['hour']),
+            int(t['minute']), int(t['second']))
+
+
+def error(s):
+    print(s)
+    sys.exit()
+
+
+class App(BaseApp):
+    report_orm = Report
+
+    def __init__(self, argv):
+        super().__init__(argv)
+        if not self.pid:
+            return
+        factory = self.get_factory('pbb_db_url')
+        self.pbb_session = factory()
+        self.base_q_log = self.prod_session.query(Log)
+        self.models = get_module_object(self.conf['models'])
+        self.PembayaranSppt = self.models.PembayaranSppt
+        self.Sppt = self.models.Sppt
+        self.base_q_psppt = self.pbb_session.query(self.PembayaranSppt)
+        self.base_q_psppt = self.base_q_psppt.filter_by(
+                kd_kanwil=self.conf['kd_kanwil'],
+                kd_kantor=self.conf['kd_kantor'], kd_tp=self.conf['kd_tp'])
+        self.base_q_sppt = self.pbb_session.query(self.Sppt)
+
+    def get_db_session(self):  # Override
+        factory = self.get_factory('db_url')
+        return factory()
+
+    def get_report(self, d):  # Override
+        return super().get_report(d['bit_011'], d['bit_037'])
+
+    def get_psppt(self, pay):
+        q = self.base_q_psppt.filter_by(
+                kd_propinsi=pay.propinsi,
+                kd_dati2=pay.kabupaten,
+                kd_kecamatan=pay.kecamatan,
+                kd_kelurahan=pay.kelurahan,
+                kd_blok=pay.blok,
+                no_urut=pay.urut,
+                kd_jns_op=pay.jenis,
+                thn_pajak_sppt=str(pay.tahun),
+                pembayaran_sppt_ke=pay.ke)
+        return q.first()
+
+    def get_sppt(self, psppt):
+        q = self.base_q_sppt.filter_by(
+                kd_propinsi=psppt.kd_propinsi,
+                kd_dati2=psppt.kd_dati2,
+                kd_kecamatan=psppt.kd_kecamatan,
+                kd_kelurahan=psppt.kd_kelurahan,
+                kd_blok=psppt.kd_blok,
+                no_urut=psppt.no_urut,
+                kd_jns_op=psppt.kd_jns_op,
+                thn_pajak_sppt=psppt.thn_pajak_sppt)
+        return q.first()
+
+    def __get_query_log(self, last_id):
+        return self.base_q_log.filter(Log.id > last_id).order_by(Log.id)
+
+    def __log_not_found(self, row, d):
+        msg = ERR_NOT_FOUND.format(invoice_id=d['nomor_bayar'], ke=row.ke)
+        self.log.error(msg)
+
+    def __log_progress(self, s_tgl, d):
+        self.log.info(f'Tgl bayar {s_tgl}, {d}')
+
+    def __create_data(self, iso, d, tgl_bayar, psppt):
+        sppt = self.get_sppt(psppt)
+        nama_wp = get_nama_wp(iso)
+        discount = hasattr(psppt, 'discount') and psppt.discount or None
+        return dict(
+            stan=d['stan'], ntb=d['ntb'],
+            kd_propinsi=psppt.kd_propinsi, kd_dati2=psppt.kd_dati2,
+            kd_kecamatan=psppt.kd_kecamatan, kd_kelurahan=psppt.kd_kelurahan,
+            kd_blok=psppt.kd_blok, no_urut=psppt.no_urut,
+            kd_jns_op=psppt.kd_jns_op, thn_pajak_sppt=psppt.thn_pajak_sppt,
+            pembayaran_sppt_ke=psppt.pembayaran_sppt_ke,
+            jml_sppt_yg_dibayar=iso['bit_004'],
+            denda_sppt=psppt.denda_sppt, tgl_pembayaran_sppt=tgl_bayar.date(),
+            tgl_rekam_byr_sppt=tgl_bayar, nm_wp_sppt=nama_wp,
+            channel_kode=iso['bit_032'].strip(), channel_nama=d['channel'],
+            pbb_yg_harus_dibayar_sppt=sppt.pbb_yg_harus_dibayar_sppt,
+            discount=discount, bank_id=iso['bit_032'], tgl_inquiry=tgl_bayar)
+
+    def __run_payment(self):
+        last = self.get_last_id('pbb json payment last id')
+        q_log = self.__get_query_log(last.as_int())
+        found = False
+        for row in q_log.limit(1000):
+            iso = eval(row.iso_request)
+            if self.get_report(iso):
+                continue
+            d = get_keys(iso)
+            psppt = self.get_psppt(row)
+            if not psppt:
+                self.__log_not_found(row, d)
+                continue
+            tgl_bayar = get_tgl_bayar(iso, psppt.tgl_rekam_byr_sppt.year)
+            s_tgl = dmyhms(tgl_bayar)
+            self.__log_progress(s_tgl, d)
+            d = self.__create_data(iso, d, tgl_bayar, psppt)
+            rpt = Report(**d)
+            last.nilai = str(row.id)
+            with transaction.manager:
+                self.rpt_session.add(rpt)
+                self.rpt_session.add(last)
+            found = True
+        return found
+
+    def __update_from_id(self):
+        q_log = self.__get_query_log(self.last_id)
+        found = False
+        for row_pay in q_log.limit(1000):
+            iso = eval(row_pay.iso_request)
+            d = get_keys(iso)
+            psppt = self.get_psppt(row_pay)
+            if not psppt:
+                self.__log_not_found(row_pay, d)
+                continue
+            tgl_bayar = get_tgl_bayar(iso, psppt.tgl_rekam_byr_sppt.year)
+            s_tgl = dmyhms(tgl_bayar)
+            self.__log_progress(s_tgl, d)
+            d = self.__create_data(iso, d, tgl_bayar, psppt)
+            rpt = self.get_report(iso)
+            if not rpt:
+                rpt = Pbb()
+            rpt.from_dict(d)
+            with transaction.manager:
+                self.rpt_session.add(rpt)
+            found = True
+            self.last_id = row_pay.id
+        return found
+
+    def run_payment(self):  # Override
+        if self.option.update_from_id:
+            self.last_id = self.option.update_from_id
+            func = self.__update_from_id
+        elif self.option.update_from_date:
+            tgl = date_from_str(self.option.update_from_date)
+            Psppt = self.PembayaranSppt
+            q = self.base_q_psppt.filter(Psppt.tgl_rekam_byr_sppt >= tgl)
+            row_psppt = q.order_by(Psppt.tgl_rekam_byr_sppt).first()
+            if not row_psppt:
+                error('Kosong')
+            invoice_id = get_id(row_psppt)
+            print(f'DEBUG Invoice ID {invoice_id}')
+            q = self.base_q_log.filter_by(
+                    propinsi=row_psppt.kd_propinsi,
+                    kabupaten=row_psppt.kd_dati2,
+                    kecamatan=row_psppt.kd_kecamatan,
+                    kelurahan=row_psppt.kd_kelurahan,
+                    blok=row_psppt.kd_blok,
+                    urut=row_psppt.no_urut,
+                    jenis=row_psppt.kd_jns_op,
+                    tahun=row_psppt.thn_pajak_sppt,
+                    ke=row_psppt.pembayaran_sppt_ke)
+            row_log = q.order_by(Log.id).first()
+            self.last_id = row_log.id - 1
+            func = self.__update_from_id
+        else:
+            func = self.__run_payment
+        while True:
+            found = func()
+            if not found:
+                break
+
+
+def main(argv=sys.argv[1:]):
+    app = App(argv)
+    app.run()

pbb-json.ini

+[main]
+models = sismiop.models.default
+db_url = postgresql://user:pass@localhost/db
+pbb_db_url = postgresql://user:pass@localhost/db
+report_db_url = postgresql://user:pass@localhost/db
+pid_file = /home/sugiana/tmp/pbb-json-report.pid
+log_file = /home/sugiana/log/pbb-json-report.log
+kd_kanwil = 22
+kd_kantor = 14
+kd_tp = 47

pbb.ini

+[main]
+models = sismiop.models.default
+db_url = postgresql://user:pass@localhost/db
+report_db_url = postgresql://user:pass@localhost/db
+pid_file = /home/sugiana/tmp/pbb-report.pid
+log_file = /home/sugiana/log/pbb-report.log

setup.py

@@ -36,9 +36,11 @@ setup(
         'console_scripts': [
             'payment_report_init_db = payment_report.scripts.common:init_db',
             'pbb_report = payment_report.scripts.pbb:main',
+            'pbb_json_report = payment_report.scripts.pbb_json:main',
             'pospbb_report = payment_report.scripts.pospbb:main',
             'va_pbb_report = payment_report.scripts.va_pbb:main',
             'bphtb_report = payment_report.scripts.bphtb:main',
+            'bphtb_json_report = payment_report.scripts.bphtb_json:main',
             'bphtb2_report = payment_report.scripts.bphtb2:main',
             'pad_report = payment_report.scripts.pad:main',
             'pad2_report = payment_report.scripts.pad2:main',