PBB berbasis tabel pembayaran_sppt

CHANGES.txt

-0.3.3 18-5-2022
----------------
-- Script baru WEBR yang belum menggunakan tabel ISO8583
-
-0.3.2 6-3-2022
+2.0 2022-05-18
 --------------
 --------------
-- Script baru WEBR yang tidak mengaitkan dengan tabel ar_payment karena
-  transaksi melalui web service (Kota Tangerang Selatan)
-
-0.3.1 3-3-2022
---------------
-- Tambah PAD JSON (PT POS)
-
-0.3 28-2-2022
--------------
-- Tambah baris models pada file konfigurasi sebagai pengganti module
-- Perbaikan pada BPHTB terkait tabel bphtb_perolehan yang berbeda schema
-- Tambah BPHTB JSON (PT POS)
-- Tambah PBB JSON (PT POS)
-
-0.2.1 23-12-2021
-----------------
-- Tambah WEBR
-
-0.2 4-12-2021
--------------
-- Tambah field jenis_pajak untuk PAD
-- Script baru PAD yang tidak lagi membaca tabel pad_payment sebagai catatan
-  ISO8583 melainkan dari tabel log_iso dan log_pad.
-- Penggunaan pkgutil agar fleksibel saat pip install -e
-
-0.1.5 16-6-2021
----------------
-- Bug fixed saat tabel pembayaran_sppt tidak memiliki field discount
-- Log PBB disertai nilai bit terkait untuk memudahkan analisa nama channel
-
-0.1.4 27-5-2021
----------------
-- Tambah channel BAYARIN
-
-0.1.3 30-4-2021
----------------
-- Tambah channel Virtual Account PBB
-- Tambah channel POSPBB 
-- Tambah opsi --update-from-id dan --update-from-date
-- Perbaikan channel
-
-0.1.2 13-4-2021
----------------
-- Penambahan models pada konfigurasi PBB yang berisi nama modul untuk ORM Sppt
-  dan PembayaranSppt 
-- Penambahan field pada PBB
-- Pada PBB field user_id boleh NULL 
-
-0.1.1 3-4-2021
---------------
-- Tambah PBB
-- Tidak ada lagi bin/pad_report_init_db dan bin/bphtb_report_init_db, sebagai
-  gantinya adalah bin/payment_report_init_db.
-- Tambah BPHTB "versi 2" yang hanya menggunakan tabel log_iso ketimbang tabel
-  bphtb_payment.
-
-0.1 2-4-2021
-------------
-- Kali pertama
+- Perbedaan utama dari versi 0.x adalah kini berangkat dari tabel aslinya dan
+  bukan dari tabel ISO8583 karena sudah memperhatikan channel MANUAL (input
+  manual pembayaran)

payment_report/models.py

@@ -66,17 +66,16 @@ class Pad(Base, Common):
 
 
 class Webr(Base, Common):
 class Webr(Base, Common):
     __tablename__ = 'webr_report'
     __tablename__ = 'webr_report'
+    # webr.ar_payment.id
     id = Column(Integer, primary_key=True)
     id = Column(Integer, primary_key=True)
     # Bit 11
     # Bit 11
-    stan = Column(String(6), nullable=False)
+    stan = Column(String(6))
     # Bit 48
     # Bit 48
-    ntb = Column(String(32), nullable=False)
+    ntb = Column(String(32))
     # log_iso.created
     # log_iso.created
     tgl = Column(Date, nullable=False)
     tgl = Column(Date, nullable=False)
     # log_iso.created
     # log_iso.created
     jam = Column(Time, nullable=False)
     jam = Column(Time, nullable=False)
-    # webr.ar_payment.id
-    payment_id = Column(Integer, nullable=False)
     # Bit 61
     # Bit 61
     nomor_bayar = Column(String(16), nullable=False)
     nomor_bayar = Column(String(16), nullable=False)
     # webr.ar_invoice.subjek_nama
     # webr.ar_invoice.subjek_nama
@@ -93,9 +92,6 @@ class Webr(Base, Common):
     channel_name = Column(String(32), nullable=False)
     channel_name = Column(String(32), nullable=False)
     # pad_reversal.tgl
     # pad_reversal.tgl
     tgl_batal = Column(DateTime(timezone=True))
     tgl_batal = Column(DateTime(timezone=True))
-    __table_args__ = (
-        UniqueConstraint('stan', 'ntb'),
-        )
 
 
 
 
 class Bphtb(Base, Common):
 class Bphtb(Base, Common):
@@ -146,30 +142,30 @@ class Pbb(Base, Common):
     __tablename__ = 'pbb_report'
     __tablename__ = 'pbb_report'
     id = Column(Integer, primary_key=True)
     id = Column(Integer, primary_key=True)
     # Bit 11
     # Bit 11
-    stan = Column(String(6), nullable=False)
+    stan = Column(String(6))
     # Bit 48
     # Bit 48
-    ntb = Column(String(32), nullable=False)
-    # Bit 62 / Invoice ID
+    ntb = Column(String(32))
+    # Field pembayaran_sppt.kd_propinsi
     kd_propinsi = Column(String(2), nullable=False)
     kd_propinsi = Column(String(2), nullable=False)
-    # Bit 62 / Invoice ID
+    # Field pembayaran_sppt.kd_dati2
     kd_dati2 = Column(String(2), nullable=False)
     kd_dati2 = Column(String(2), nullable=False)
-    # Bit 62 / Invoice ID
+    # Field pembayaran_sppt.kd_kecamatan
     kd_kecamatan = Column(String(3), nullable=False)
     kd_kecamatan = Column(String(3), nullable=False)
-    # Bit 62 / Invoice ID
+    # Field pembayaran_sppt.kd_kelurahan
     kd_kelurahan = Column(String(3), nullable=False)
     kd_kelurahan = Column(String(3), nullable=False)
-    # Bit 62 / Invoice ID
+    # Field pembayaran_sppt.kd_blok
     kd_blok = Column(String(3), nullable=False)
     kd_blok = Column(String(3), nullable=False)
-    # Bit 62 / Invoice ID
+    # Field pembayaran_sppt.no_urut
     no_urut = Column(String(4), nullable=False)
     no_urut = Column(String(4), nullable=False)
-    # Bit 62 / Invoice ID
+    # Field pembayaran_sppt.kd_jns_op
     kd_jns_op = Column(String(1), nullable=False)
     kd_jns_op = Column(String(1), nullable=False)
-    # Bit 62 / Invoice ID
+    # Field pembayaran_sppt.thn_pajak_sppt
     thn_pajak_sppt = Column(String(4), nullable=False)
     thn_pajak_sppt = Column(String(4), nullable=False)
-    # Field sppt.pbb_yg_harus_dibayar_sppt
-    pbb_yg_harus_dibayar_sppt = Column(Float, nullable=False)
     # Field pembayaran_sppt.pembayaran_sppt_ke
     # Field pembayaran_sppt.pembayaran_sppt_ke
     pembayaran_sppt_ke = Column(Integer, nullable=False)
     pembayaran_sppt_ke = Column(Integer, nullable=False)
-    # Bit 4
+    # Field sppt.pbb_yg_harus_dibayar_sppt
+    pbb_yg_harus_dibayar_sppt = Column(Float, nullable=False)
+    # Field pembayaran_sppt.jml_sppt_yg_dibayar
     jml_sppt_yg_dibayar = Column(BigInteger, nullable=False)
     jml_sppt_yg_dibayar = Column(BigInteger, nullable=False)
     # Field pembayaran_sppt.denda_sppt
     # Field pembayaran_sppt.denda_sppt
     denda_sppt = Column(BigInteger, nullable=False)
     denda_sppt = Column(BigInteger, nullable=False)
@@ -193,5 +189,8 @@ class Pbb(Base, Common):
     bank_id = Column(String(4))
     bank_id = Column(String(4))
     tgl_batal = Column(DateTime(timezone=True))
     tgl_batal = Column(DateTime(timezone=True))
     __table_args__ = dict(
     __table_args__ = dict(
-        UniqueConstraint('stan', 'ntb'),
+        UniqueConstraint(
+            'kd_propinsi', 'kd_dati2', 'kd_kecamatan', 'kd_kelurahan',
+            'kd_blok', 'no_urut', 'kd_jns_op', 'thn_pajak_sppt'),
+
         )
         )

payment_report/scripts/common.py

@@ -228,8 +228,6 @@ def make_pid_file(filename):
 
 
 
 
 class BaseApp:
 class BaseApp:
-    report_orm = None  # Override, please
-
     def __init__(self, argv):
     def __init__(self, argv):
         self.option = get_option(argv)
         self.option = get_option(argv)
         cp = ConfigParser()
         cp = ConfigParser()
@@ -247,7 +245,6 @@ class BaseApp:
         factory = self.get_factory('report_db_url')
         factory = self.get_factory('report_db_url')
         self.rpt_session = factory()
         self.rpt_session = factory()
         register(self.rpt_session)
         register(self.rpt_session)
-        self.base_q_report = self.rpt_session.query(self.report_orm)
 
 
     def get_factory(self, name):
     def get_factory(self, name):
         db_url = self.conf[name]
         db_url = self.conf[name]

payment_report/scripts/data/conf.csv

@@ -5,7 +5,6 @@ pad json payment last id,0,pad_payment.id terakhir yang diproses
 pad json reversal last date,1-1-2000 00:00:00,pad_reversal.tgl terakhir yang diproses
 pad json reversal last date,1-1-2000 00:00:00,pad_reversal.tgl terakhir yang diproses
 pad2 payment last id,0,log_iso.id terakhir yang diproses
 pad2 payment last id,0,log_iso.id terakhir yang diproses
 pad2 reversal last id,0,log_iso.id terakhir yang diproses
 pad2 reversal last id,0,log_iso.id terakhir yang diproses
-webr0 payment last id,0,webr.ar_payment.id terakhir yang diproses
 webr payment last id,0,log_iso.id terakhir yang diproses
 webr payment last id,0,log_iso.id terakhir yang diproses
 webr2 payment last id,0,log_iso.id terakhir yang diproses
 webr2 payment last id,0,log_iso.id terakhir yang diproses
 webr2 reversal last id,0,log_iso.id terakhir yang diproses
 webr2 reversal last id,0,log_iso.id terakhir yang diproses
@@ -15,7 +14,7 @@ bphtb json payment last id,0,bphtb.bphtb_payment.id terakhir yang diproses
 bphtb json reversal last date,1-1-2000 00:00:00,bphtb.bphtb_reversal.tgl terakhir yang diproses
 bphtb json reversal last date,1-1-2000 00:00:00,bphtb.bphtb_reversal.tgl terakhir yang diproses
 bphtb2 payment last id,0,log_iso.id terakhir yang diproses
 bphtb2 payment last id,0,log_iso.id terakhir yang diproses
 bphtb2 reversal last id,0,log_iso.id terakhir yang diproses
 bphtb2 reversal last id,0,log_iso.id terakhir yang diproses
-pbb payment last id,0,inquiry.id terakhir yang diproses
+pbb payment last date,1-1-2000,pembayaran_sppt.tgl_rekam_byr_sppt terakhir yang diproses
 pbb reversal last date,1-1-2000 00:00:00,reversal.tgl terakhir yang diproses
 pbb reversal last date,1-1-2000 00:00:00,reversal.tgl terakhir yang diproses
 pbb2 payment last id,0,log_iso.id terakhir yang diproses
 pbb2 payment last id,0,log_iso.id terakhir yang diproses
 pbb2 reversal last id,0,log_iso.id terakhir yang diproses
 pbb2 reversal last id,0,log_iso.id terakhir yang diproses

payment_report/scripts/tools.py

+from datetime import (
+    datetime,
+    date,
+    time as mktime,
+    )
+
+
+#################
+# Compare value #
+#################
+MIDNIGHT_TIME = mktime(0, 0, 0)
+
+
+def split_time(t):
+    if isinstance(t, datetime):
+        return t.date(), t.time()
+    return t, MIDNIGHT_TIME
+
+
+def is_same(a, b):
+    if a == b:
+        return True
+    if not (isinstance(a, date) or isinstance(a, datetime)):
+        return False
+    date_a, time_a = split_time(a)
+    date_b, time_b = split_time(b)
+    if date_a != date_b:
+        return False
+    if isinstance(a, date) or isinstance(b, date):
+        return True
+    if time_a == time_b:
+        return True
+    return False

payment_report/scripts/webr.py

 import sys
 import sys
+from datetime import datetime
 from sqlalchemy.exc import ProgrammingError
 from sqlalchemy.exc import ProgrammingError
 import transaction
 import transaction
 from opensipkd.waktu import dmyhms
 from opensipkd.waktu import dmyhms
@@ -6,98 +7,122 @@ from opensipkd.webr.models.default import (
     Payment,
     Payment,
     Invoice,
     Invoice,
     )
     )
+from iso8583_web.models.meta import Base as BaseConf
 from opensipkd.iso8583.bjb.webr.structure import PAYMENT_CODE
 from opensipkd.iso8583.bjb.webr.structure import PAYMENT_CODE
 from opensipkd.iso8583.bjb.webr.models import (
 from opensipkd.iso8583.bjb.webr.models import (
     LogMixin,
     LogMixin,
     Log,
     Log,
     )
     )
-from iso8583_web.models.meta import Base as BaseConf
 from ..models import (
 from ..models import (
     Base,
     Base,
     Webr,
     Webr,
     )
     )
 from .common import (
 from .common import (
-    App2 as BaseApp,
+    BaseApp,
     init_db as base_init_db,
     init_db as base_init_db,
+    BIT_18_NAMES,
+    get_channel_name_by_row,
     )
     )
 
 
 
 
-ERR_NOT_FOUND = 'Tgl {tgl_bayar} nomor bayar {invoice_id} tidak ada'
-
-
 class AlternativeLog(Base, LogMixin):
 class AlternativeLog(Base, LogMixin):
     __table_args__ = dict(schema='webr')
     __table_args__ = dict(schema='webr')
 
 
 
 
 class App(BaseApp):
 class App(BaseApp):
-    field_invoice_id = 'bit_061'
-    field_ntb = 'bit_048'
-    report_orm = Webr
-    log_orm = Log
-
     def __init__(self, argv):
     def __init__(self, argv):
         super().__init__(argv)
         super().__init__(argv)
         if not self.pid:
         if not self.pid:
             return
             return
-        self.base_q_iso_resp = self.prod_session.query(Log)
+        self.base_q_pay = self.prod_session.query(Payment, Invoice).filter(
+                Payment.ar_invoice_id == Invoice.id)
+        self.base_q_iso = self.prod_session.query(Log)
         try:
         try:
-            self.base_q_iso_resp.first()
+            self.base_q_iso.first()
+            self.IsoLog = Log
         except ProgrammingError:
         except ProgrammingError:
             self.prod_session.rollback()
             self.prod_session.rollback()
-            self.base_q_iso_resp = self.prod_session.query(AlternativeLog)
-        self.base_q_iso_resp = self.base_q_iso_resp.filter_by(
-                mti='0210', bit_003=PAYMENT_CODE)
-        self.base_q_pay = self.prod_session.query(Payment, Invoice).filter(
-                Payment.ar_invoice_id == Invoice.id)
+            self.base_q_iso = self.prod_session.query(AlternativeLog)
+            self.IsoLog = AlternativeLog
+        self.base_q_iso = self.base_q_iso.filter_by(
+                mti='0210', bit_003=PAYMENT_CODE, bit_039='00')
 
 
-    def get_pay(self, d):
-        q_pay = self.base_q_pay.filter(Invoice.kode == d['nomor_bayar'])
-        return q_pay.order_by(Payment.id.desc()).first()
+    def get_channel(self, pay, inv):
+        if not pay.bank_id:
+            return '0000', 'MANUAL', None
+        nomor_bayar = inv.kode
+        q = self.base_q_iso.filter(self.IsoLog.bit_061.like(f'{nomor_bayar}%'))
+        iso = q.first()
+        if not iso:
+            name = BIT_18_NAMES[str(pay.channel_id)]
+            return pay.channel_id, name, None
+        name = get_channel_name_by_row(iso)
+        return iso.bit_018, name, iso
 
 
-    def is_iso_resp_ok(self, iso_req):
-        q = self.base_q_iso_resp.filter_by(
-                bit_011=iso_req.bit_011, bit_048=iso_req.bit_048)
-        iso_resp = q.order_by(Log.id.desc()).first()
-        return iso_resp and iso_resp.bit_039 == '00'
+    def update_last(self, pay):
+        self.last_id = pay.id
+        self.last.nilai = str(pay.id)
+        with transaction.manager:
+            self.rpt_session.add(self.last)
 
 
-    def run_payment(self):
-        last = self.get_last_id('webr payment last id')
-        q_iso_req = self.prod_session.query(Log).filter_by(
-                    mti='0200', bit_003=PAYMENT_CODE).filter(
-                    Log.id > last.as_int())
-        for iso_req in q_iso_req.order_by(Log.id):
-            if self.get_report(iso_req):
-                continue
-            if not self.is_iso_resp_ok(iso_req):
-                continue
-            d = self.get_keys(iso_req)
-            tgl_bayar = iso_req.created.date()
-            pay = self.get_pay(d)
-            if not pay:
-                msg = ERR_NOT_FOUND.format(
-                        tgl_bayar=tgl_bayar, invoice_id=d['nomor_bayar'])
-                self.log.error(msg)
+    def update_from_id(self):
+        q_pay = self.base_q_pay.filter(Payment.id > self.last_id)
+        no = 0
+        for pay, inv in q_pay.order_by(Payment.id).limit(1000):
+            no += 1
+            channel_id, channel_name, iso = self.get_channel(pay, inv)
+            nomor_bayar = inv.kode
+            tgl_bayar = pay.created.date()
+            s_tgl = dmyhms(pay.created)
+            stan = iso and iso.bit_011 or None
+            ntb = pay.ntb and pay.ntb.strip() or None
+            msg = f'#{no} ID {pay.id}, Tgl bayar {s_tgl}, '\
+                  f'Nomor bayar {nomor_bayar}, Channel {channel_name}'
+            if iso:
+                msg = f'{msg}, STAN {stan}, NTB {ntb}'
+            if not pay.bayar:
+                self.log.warning(f'{msg}, field bayar 0, abaikan')
+                self.last and self.update_last(pay)
                 continue
                 continue
-            pay, inv = pay
-            s_tgl = dmyhms(iso_req.created)
-            self.log.info(
-                f'Tgl bayar {s_tgl}, Nomor bayar {d["nomor_bayar"]}, '
-                f'STAN {d["stan"]}, NTB {d["ntb"]}, Channel {d["channel"]}')
-            rpt = Webr(
-                    stan=d['stan'], ntb=d['ntb'], tgl=tgl_bayar,
-                    jam=iso_req.created.time(), payment_id=pay.id,
-                    nomor_bayar=d['nomor_bayar'],
-                    nama_wp=inv.subjek_nama, pokok=inv.jumlah-inv.bunga,
-                    denda=pay.bunga, jml_bayar=pay.bayar,
-                    channel_id=iso_req.bit_018.strip(),
-                    channel_name=d['channel'])
-            last.nilai = str(iso_req.id)
+            self.log.info(msg)
+            d = dict(
+                    stan=stan, ntb=ntb, tgl=tgl_bayar, jam=pay.created.time(),
+                    nomor_bayar=nomor_bayar, nama_wp=inv.subjek_nama,
+                    pokok=inv.jumlah-inv.bunga, denda=pay.bunga,
+                    jml_bayar=pay.bayar, channel_id=channel_id,
+                    channel_name=channel_name)
+            q = self.rpt_session.query(Webr).filter_by(id=pay.id)
+            rpt = q.first()
+            if rpt:
+                rpt.from_dict(d)
+            else:
+                d['id'] = pay.id
+                rpt = Webr(**d)
+            self.last_id = pay.id
             with transaction.manager:
             with transaction.manager:
                 self.rpt_session.add(rpt)
                 self.rpt_session.add(rpt)
-                self.rpt_session.add(last)
+                if self.last:
+                    self.last.nilai = str(pay.id)
+                    self.rpt_session.add(last)
+        return no
 
 
-    def run_reversal(self):  # Override
-        super().run_reversal('webr reversal last id')
+    def run_payment(self):  # Override
+        self.last = None
+        if self.option.update_from_id:
+            self.last_id = self.option.update_from_id
+        elif self.option.update_from_date:
+            tgl = datetime.strptime(self.option.update_from_date, '%d-%m-%Y')
+            q = self.prod_session.query(Payment).filter(
+                    Payment.created >= tgl).order_by(Payment.id)
+            row = q.first()
+            self.last_id = row.id - 1
+        else:
+            self.last = self.get_last_id('webr payment last id')
+            self.last_id = self.last.as_int()
+        while True:
+            found = self.update_from_id()
+            if not found:
+                break
 
 
 
 
 def main(argv=sys.argv[1:]):
 def main(argv=sys.argv[1:]):

payment_report/scripts/webr0.py

-import sys
-import transaction
-from opensipkd.waktu import dmyhms
-from opensipkd.webr.models.default import (
-    Payment,
-    Invoice,
-    )
-from iso8583_web.models.meta import Base as BaseConf
-from opensipkd.iso8583.bjb.webr.models import Log
-from ..models import (
-    Base,
-    Webr,
-    )
-from .common import (
-    BaseApp,
-    init_db as base_init_db,
-    BIT_18_NAMES,
-    )
-
-
-class App(BaseApp):
-    report_orm = Webr
-
-    def __init__(self, argv):
-        super().__init__(argv)
-        if not self.pid:
-            return
-        self.base_q_pay = self.prod_session.query(Payment, Invoice).filter(
-                Payment.ar_invoice_id == Invoice.id)
-
-    def tgl_awal_log_iso(self):
-        q = self.prod_session.query(Log).order_by(Log.id)
-        row = q.first()
-        return row.created.date()
-
-    def run_payment(self):
-        tgl = self.tgl_awal_log_iso()
-        last = self.get_last_id('webr0 payment last id')
-        q_pay = self.base_q_pay.filter(
-                    Payment.created < tgl, Payment.id > last.as_int())
-        rpt_id = 0
-        for pay, inv in q_pay.order_by(Payment.id):
-            rpt_id += 1
-            if not pay.channel_id:  # Biasanya hasil test
-                continue
-            nomor_bayar = inv.kode
-            tgl_bayar = pay.created.date()
-            s_tgl = dmyhms(pay.created)
-            stan = pay.created.strftime('%H%M%S')
-            channel_id = str(pay.channel_id)
-            channel = BIT_18_NAMES[channel_id]
-            self.log.info(
-                f'Tgl bayar {s_tgl}, Nomor bayar {nomor_bayar}, '
-                f'STAN {stan}, NTB {pay.ntb}, Channel {channel}')
-            rpt = Webr(
-                    id=rpt_id, stan=stan, ntb=pay.ntb, tgl=tgl_bayar,
-                    jam=pay.created.time(), payment_id=pay.id,
-                    nomor_bayar=nomor_bayar, nama_wp=inv.subjek_nama,
-                    pokok=inv.jumlah-inv.bunga, denda=pay.bunga,
-                    jml_bayar=pay.bayar, channel_id=channel_id,
-                    channel_name=channel)
-            last.nilai = str(pay.id)
-            with transaction.manager:
-                self.rpt_session.add(rpt)
-                self.rpt_session.add(last)
-
-
-def main(argv=sys.argv[1:]):
-    app = App(argv)
-    if app.pid:
-        app.run()
-
-
-def init_db(argv=sys.argv[1:]):
-    base_init_db(Base.metadata, BaseConf.metadata, argv)

setup.py

@@ -45,7 +45,6 @@ setup(
             'pad_report = payment_report.scripts.pad:main',
             'pad_report = payment_report.scripts.pad:main',
             'pad_json_report = payment_report.scripts.pad_json:main',
             'pad_json_report = payment_report.scripts.pad_json:main',
             'pad2_report = payment_report.scripts.pad2:main',
             'pad2_report = payment_report.scripts.pad2:main',
-            'webr0_report = payment_report.scripts.webr0:main',
             'webr_report = payment_report.scripts.webr:main',
             'webr_report = payment_report.scripts.webr:main',
             'webr2_report = payment_report.scripts.webr2:main',
             'webr2_report = payment_report.scripts.webr2:main',
             ],
             ],