Kali pertama

.gitignore

+test*

CHANGES.txt

+0.1 2-4-2021
+------------
+- Kali pertama

MANIFEST.in

+include CHANGES.txt devel.ini README.rst
+recursive-include payment_report *.py *.csv

README.rst

+Pembuatan Tabel Laporan
+=======================
+
+Paket ini bertujuan untuk menyalin data pembayaran dari berbagai database ke
+tabel lain yang akan digunakan oleh web report.
+
+Pasang paket yang dibutuhkan::
+
+    $ python3.9 -m venv ~/env
+    $ ~/env/bin/pip install --upgrade pip
+    $ ~/env/bin/pip install wheel
+    $ ~/env/bin/pip install ebcdic
+    $ ~/env/bin/pip install git+https://git.opensipkd.com/sugiana/opensipkd-iso8583-bjb.git
+
+Buatlah database ``payment_report``::
+
+    $ sudo su - postgres -c "createuser -P report"
+    $ sudo su - postgres -c "createdb -O report payment_report"
+
+Misalkan database production bernama ``simpad`` maka berilah hak akses yang
+diperlukan user ``report``::
+
+    $ sudo su - postgres -c "psql simpad"
+
+    GRANT USAGE ON SCHEMA pad TO report;
+    GRANT SELECT ON pad.pad_sspd TO report;
+    GRANT SELECT ON pad.pad_spt TO report;
+    GRANT SELECT ON pad.pad_pajak TO report;
+    GRANT SELECT ON pad.pad_customer TO report;
+    GRANT SELECT ON pad.pad_customer_usaha TO report;
+    GRANT SELECT ON pad_payment TO report;
+    GRANT SELECT ON pad_reversal TO report;
+
+Lalu buat file konfigurasi bernama ``pad.ini``::
+
+    [main]
+    module = opensipkd.iso8583.bjb.pad.default
+    db_url = postgresql://report:password@localhost:5432/simpad
+    report_db_url = postgresql://report:password@localhost:5432/payment_report
+    pid_file = /home/sugiana/tmp/pad-report.pid
+    log_file = /home/sugiana/log/pad-report.log
+
+Buat tabel yang dibutuhkan di database ``payment_report``::
+
+    $ ~/env/bin/pad_report_init_db pad.ini
+
+Kemudian buat file ``/etc/cron.d/payment-report`` berikut ini::
+
+    MAILTO=""
+    */5 * * * * sugiana cd /home/sugiana/payment-report && ../env/bin/pad_report pad.ini
+
+Lalu pantau log-nya. Jika sudah ada penampakan lihat hasilnya di tabel ``pad_report``.
+
+
+BPHTB
+-----
+
+Berikan hak akses yang diperlukan user ``report``::
+
+    $ sudo su - postgres -c "psql bphtb"
+
+    GRANT USAGE ON SCHEMA bphtb TO report;
+    GRANT SELECT ON bphtb.bphtb_payment TO report;
+    GRANT SELECT ON bphtb.bphtb_reversal TO report;
+    GRANT SELECT ON bphtb.bphtb_bank TO report;
+    GRANT SELECT ON bphtb.bphtb_sspd TO report;
+    GRANT SELECT ON bphtb.bphtb_perolehan TO report;
+    GRANT SELECT ON bphtb.bphtb_ppat TO report;
+
+Lalu buat file konfigurasi bernama ``bphtb.ini``::
+
+    [main]
+    module = opensipkd.iso8583.bjb.bphtb.default
+    db_url = postgresql://report:password@localhost:5434/bphtb
+    report_db_url = postgresql://report:password@localhost:5434/payment_report
+    pid_file = /home/sugiana/tmp/bphtb-report.pid
+    log_file = /home/sugiana/log/bphtb-report.log
+
+Buat tabel yang dibutuhkan di database ``payment_report``::
+
+    $ ~/env/bin/bphtb_report_init_db bphtb.ini
+
+Kemudian di file ``/etc/cron.d/payment-report`` tambahkan::
+
+    */5 * * * * sugiana cd /home/sugiana/payment-report && ../env/bin/bphtb_report bphtb.ini
+
+Lalu pantau log-nya. Jika sudah ada penampakan lihat hasilnya di tabel ``bphtb_report``.
+
+
+

payment_report/models.py

+from sqlalchemy import (
+    Column,
+    Integer,
+    Float,
+    String,
+    Date,
+    Time,
+    DateTime,
+    UniqueConstraint,
+    )
+from sqlalchemy.ext.declarative import declarative_base
+
+
+Base = declarative_base()
+
+
+class Pad(Base):
+    __tablename__ = 'pad_report'
+    id = Column(Integer, primary_key=True)
+    # Bit 11
+    stan = Column(String(6), nullable=False)
+    # Bit 48
+    ntb = Column(String(32), nullable=False)
+    # pad.pad_sspd.create_date
+    tgl = Column(Date, nullable=False)
+    # pad.pad_sspd.sspdjam
+    jam = Column(Time, nullable=False)
+    # pad.pad_sspd.id
+    sspd_id = Column(Integer, nullable=False)
+    # Bit 61
+    nomor_bayar = Column(String(16), nullable=False)
+    # pad.pad_pajak.masapajak
+    masa_pajak = Column(Integer, nullable=False)
+    # pad.pad_customer
+    npwpd = Column(String(17), nullable=False)
+    # pad.pad_customer.customernm
+    nama_wp = Column(String(150), nullable=False)
+    # pad.pad_sspd.jml_bayar - pad.pad_sspd.denda
+    pokok = Column(Float, nullable=False)
+    # pad.pad_sspd.denda
+    denda = Column(Float, nullable=False)
+    # pad.pad_sspd.jml_bayar
+    jml_bayar = Column(Float, nullable=False)
+    # Bit 18
+    channel_id = Column(String(4), nullable=False)
+    # Bit 41 / 42 / 43
+    channel_name = Column(String(32), nullable=False)
+    # pad_reversal.tgl
+    tgl_batal = Column(DateTime(timezone=True))
+    __table_args__ = (
+        UniqueConstraint('stan', 'ntb'),
+        )
+
+
+class Bphtb(Base):
+    __tablename__ = 'bphtb_report'
+    id = Column(Integer, primary_key=True)
+    # Bit 11
+    stan = Column(String(6), nullable=False)
+    # Bit 58
+    ntb = Column(String(32), nullable=False)
+    # bphtb.bphtb_bank.tanggal
+    tgl = Column(Date, nullable=False)
+    # bphtb.bphtb_bank.jam
+    jam = Column(Time, nullable=False)
+    # Bit 62
+    invoice_id = Column(String(16), nullable=False)
+    # Bit 61
+    nop = Column(String(22), nullable=False)
+    # bphtb.bphtb_bank.wp_nama
+    wp_nama = Column(String(50), nullable=False)
+    # bphtb.bphtb_bank.wp_alamat
+    wp_alamat = Column(String(100), nullable=False)
+    # bphtb.bphtb_sspd.op_alamat
+    op_alamat = Column(String(100), nullable=False)
+    # bphtb.bphtb_bank.npop
+    npop = Column(Float, nullable=False)
+    # bphtb.bphtb_bank.bumi_luas
+    bumi_luas = Column(Integer, nullable=False)
+    # bphtb.bphtb_bank.bng_luas
+    bng_luas = Column(Integer, nullable=False)
+    # Bit 4
+    nilai_bphtb = Column(Float, nullable=False)
+    # bphtb.bphtb_perolehan.nama
+    jenis_perolehan = Column(String(100), nullable=False)
+    # bphtb.bphtb_ppat.nama
+    ppat = Column(String(50), nullable=False)
+    # Bit 18
+    channel_id = Column(String(4), nullable=False)
+    # Bit 41 / 42 / 43
+    channel_nama = Column(String(32), nullable=False)
+    # bphtb.bphtb_reversal.tgl
+    tgl_batal = Column(DateTime(timezone=True))
+    __table_args__ = (
+        UniqueConstraint('stan', 'ntb'),
+        )

payment_report/scripts/bphtb.py

+import sys
+import transaction
+from opensipkd.waktu import dmyhms
+from opensipkd.bphtb.models.default import (
+    IsoPayment,
+    IsoReversal,
+    Payment,
+    Invoice,
+    Perolehan,
+    Customer,
+    )
+from opensipkd.bphtb.services.base import get_db_session
+from iso8583_web.models.meta import Base as BaseConf
+from opensipkd.iso8583.bjb.bphtb import Doc
+from ..models import (
+    Base,
+    Bphtb,
+    )
+from .common import (
+    get_iso,
+    get_keys,
+    App as BaseApp,
+    init_db as base_init_db,
+    )
+
+
+class App(BaseApp):
+    iso_class = Doc
+    report_orm = Bphtb
+    iso_reversal_orm = IsoReversal
+
+    def __init__(self, argv):
+        super().__init__(argv)
+        if not self.pid:
+            return
+        self.base_q_inv = self.prod_session.query(
+                Invoice, Customer, Perolehan).filter(
+                Invoice.ppat_id == Customer.id,
+                Invoice.perolehan_id == Perolehan.id)
+
+    def get_db_session(self):  # Override
+        return get_db_session()
+
+    def run_payment(self):  # Override
+        last = self.get_last_id('bphtb payment last id')
+        q_iso = self.prod_session.query(IsoPayment, Payment).filter(
+                IsoPayment.id == Payment.id,
+                IsoPayment.id > last.as_int())
+        for row_iso, row_pay in q_iso.order_by(IsoPayment.id):
+            iso = get_iso(row_iso.iso_request, Doc, self.option.debug)
+            if self.get_report(iso):
+                continue
+            tgl_bayar = row_iso.tgl.date()
+            q_inv = self.base_q_inv.filter(Invoice.id == row_pay.sspd_id)
+            inv, ppat, perolehan = q_inv.first()
+            d = get_keys(iso)
+            s_tgl = dmyhms(row_iso.tgl)
+            self.log.info(
+                f'Tgl bayar {s_tgl}, Nomor bayar {d["nomor_bayar"]}, '
+                f'STAN {d["stan"]}, NTB {d["ntb"]}, Channel {d["channel"]}')
+            rpt = Bphtb(
+                    stan=d['stan'], ntb=d['ntb'], tgl=tgl_bayar,
+                    jam=row_iso.tgl.time(),
+                    invoice_id=iso.get_invoice_id().strip(),
+                    nop=row_pay.nop, wp_nama=row_pay.wp_nama,
+                    wp_alamat=row_pay.wp_alamat, op_alamat=inv.op_alamat,
+                    npop=row_pay.npop, bumi_luas=row_pay.bumi_luas,
+                    bng_luas=row_pay.bng_luas, nilai_bphtb=row_pay.bayar,
+                    jenis_perolehan=perolehan.nama, ppat=ppat.nama,
+                    channel_id=iso.get_channel().strip(),
+                    channel_nama=d['channel'])
+            last.nilai = str(row_iso.id)
+            print(f'last.nilai {last.nilai}')
+            with transaction.manager:
+                self.rpt_session.add(rpt)
+                self.rpt_session.add(last)
+
+    def run_reversal(self):  # Override
+        super().run_reversal('bphtb reversal last date')
+
+
+def main(argv=sys.argv[1:]):
+    app = App(argv)
+    if app.pid:
+        app.run()
+
+
+def init_db(argv=sys.argv[1:]):
+    base_init_db(Base.metadata, BaseConf.metadata, argv)

payment_report/scripts/common.py

+import sys
+import os
+import logging
+import csv
+from datetime import (
+    date,
+    datetime,
+    )
+from argparse import ArgumentParser
+from configparser import ConfigParser
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+import transaction
+from zope.sqlalchemy import register
+from opensipkd.waktu import dmyhms
+from iso8583_web.models.conf import Conf as BaseConf
+from opensipkd.iso8583.bjb.scripts.common import get_module_object
+
+
+log_format = '%(asctime)s %(levelname)s %(message)s'
+formatter = logging.Formatter(log_format)
+
+BIT_18_NAMES = {
+    '6010': 'TELLER',
+    '6011': 'ATM',
+    '6012': 'POS',
+    '6013': 'PHONE BANKING',
+    '6014': 'INTERNETBANKING',
+    '6015': 'KIOSK',
+    '6016': 'AUTODEBET',
+    '6017': 'MOBILBANKING',
+    '7012': 'PT.POS'}
+
+my_registry = dict()
+
+
+def get_file(filename):
+    base_dir = os.path.split(__file__)[0]
+    fullpath = os.path.join(base_dir, 'data', filename)
+    return open(fullpath)
+
+
+def append_csv(table, filename, keys):
+    DBSession = my_registry['db_session']
+    with get_file(filename) as f:
+        reader = csv.DictReader(f)
+        filter_ = dict()
+        for cf in reader:
+            for key in keys:
+                filter_[key] = cf[key]
+            q = DBSession.query(table).filter_by(**filter_)
+            found = q.first()
+            if found:
+                continue
+            row = table()
+            for fieldname in cf:
+                val = cf[fieldname]
+                if not val:
+                    continue
+                setattr(row, fieldname, val)
+            DBSession.add(row)
+
+
+def clean_raw(raw):
+    if raw[:2] == '\\x':
+        raw = raw[2:]
+    if raw[:4] == '3032':
+        return bytes.fromhex(raw)
+    return raw.encode('utf8')
+
+
+def get_iso(raw, iso_class, debug=False):
+    raw = clean_raw(raw)
+    iso = iso_class(debug=debug)
+    iso.setIsoContent(raw)
+    return iso
+
+
+def get_keys(iso):
+    return dict(
+        nomor_bayar=iso.get_invoice_id().strip(),
+        stan=iso.get_stan().strip(),
+        ntb=iso.get_ntb().strip(),
+        channel=get_channel_name(iso))
+
+
+def get_channel_name(iso):
+    channel_id = iso.get_channel().strip()
+    if channel_id in BIT_18_NAMES:
+        return BIT_18_NAMES[channel_id]
+    if channel_id != '6025':
+        return ''
+    bit_041 = iso.getBit(41).strip()
+    bit_042 = iso.getBit(42).strip()
+    bit_043 = iso.getBit(43).strip()
+    if bit_042 in ('TOKOPEDIA', 'BUKALAPAK', 'MASAGO'):
+        return bit_042
+    if bit_042 == 'NG':
+        return bit_041
+    if bit_043 in ('INDOMARET', 'ALFAMART'):
+        return bit_043
+    return 'LAINNYA'
+
+
+def create_log(log_file):
+    file_handler = logging.FileHandler(log_file)
+    file_handler.setFormatter(formatter)
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_handler.setFormatter(formatter)
+    log = logging.getLogger(sys.argv[0])
+    log.setLevel(logging.DEBUG)
+    log.addHandler(file_handler)
+    log.addHandler(console_handler)
+    return log
+
+
+def get_parser():
+    pars = ArgumentParser()
+    pars.add_argument('conf')
+    pars.add_argument('--debug', action='store_true')
+    pars.add_argument('--debug-sql', action='store_true')
+    return pars
+
+
+def get_option(argv):
+    pars = get_parser()
+    return pars.parse_args(argv)
+
+
+class Conf(BaseConf):
+    def as_datetime(self):
+        return datetime.strptime(self.nilai, '%d-%m-%Y %H:%M:%S')
+
+
+def is_live(pid):
+    try:
+        os.kill(pid, 0)
+    except OSError:
+        return
+    return True
+
+
+def read_pid_file(filename):
+    try:
+        f = open(filename)
+        s = f.read()
+        f.close()
+        s = s.split()
+        s = s[0]
+        return int(s)
+    except IOError:
+        return
+    except ValueError:
+        return
+    except IndexError:
+        return
+
+
+def write_pid_file(filename):
+    pid = os.getpid()
+    f = open(filename, 'w')
+    f.write(str(pid))
+    f.close()
+    return pid
+
+
+def make_pid_file(filename):
+    pid = read_pid_file(filename)
+    if pid and is_live(pid):
+        print(f'PID saya {pid} masih aktif.')
+        return
+    return write_pid_file(filename)
+
+
+class App:
+    iso_class = None  # Override, please
+    report_orm = None  # Override, please
+    iso_reversal_orm = None  # Override, please
+
+    def __init__(self, argv):
+        self.option = get_option(argv)
+        conf = ConfigParser()
+        conf.read(self.option.conf)
+        cf = conf['main']
+        self.pid = make_pid_file(cf['pid_file'])
+        if not self.pid:
+            return
+        self.log = create_log(cf['log_file'])
+        module_name = cf['module']
+        module = get_module_object(module_name)
+        module_conf = dict(cf)
+        module.init(module_conf)
+        iso = self.iso_class()
+        self.prod_session = self.get_db_session()
+        self.prod_session.bind.echo = self.option.debug_sql
+        engine = create_engine(cf['report_db_url'], echo=self.option.debug_sql)
+        factory = sessionmaker(bind=engine)
+        self.rpt_session = factory()
+        register(self.rpt_session)
+        self.base_q_report = self.rpt_session.query(self.report_orm)
+
+    def get_db_session(self):  # Override, please
+        pass
+
+    def get_last_id(self, nama):
+        q = self.rpt_session.query(Conf).filter_by(nama=nama)
+        return q.first()
+
+    def get_report(self, iso):
+        d = get_keys(iso)
+        q = self.base_q_report.filter_by(stan=d['stan'], ntb=d['ntb'])
+        return q.first()
+
+    def run_payment(self):  # Override, please
+        pass
+
+    def run_reversal(self, conf_name):  # Override, please
+        last = self.get_last_id(conf_name)
+        q = self.prod_session.query(self.iso_reversal_orm).filter(
+                self.iso_reversal_orm.tgl > last.as_datetime())
+        for row in q.order_by(self.iso_reversal_orm.tgl):
+            iso = get_iso(row.iso_request, self.iso_class, self.option.debug)
+            rpt = self.get_report(iso)
+            if not rpt or rpt.tgl_batal:
+                continue
+            rpt.tgl_batal = row.tgl
+            last.nilai = s_tgl = dmyhms(row.tgl)
+            d = get_keys(iso)
+            self.log.info(
+                f'Tgl batal {s_tgl}, Nomor bayar {d["nomor_bayar"]}, '
+                f'STAN {d["stan"]}, NTB {d["ntb"]}, Channel {d["channel"]}')
+            with transaction.manager:
+                self.rpt_session.add(rpt)
+                self.rpt_session.add(last)
+
+    def run(self):
+        self.run_payment()
+        self.run_reversal()
+
+
+def init_db(metadata_rpt, metadata_conf, argv=sys.argv[1:]):
+    conf_file = argv[0]
+    conf = ConfigParser()
+    conf.read(conf_file)
+    db_url = conf.get('main', 'report_db_url')
+    engine = create_engine(db_url, echo=True)
+    metadata_rpt.create_all(engine)
+    metadata_conf.create_all(engine)
+    factory = sessionmaker(bind=engine)
+    my_registry['db_session'] = db_session = factory()
+    register(db_session)
+    with transaction.manager:
+        append_csv(Conf, 'conf.csv', ['nama'])

payment_report/scripts/data/conf.csv

+nama,nilai,keterangan
+pad payment last id,0,pad_payment.id terakhir yang diproses
+pad reversal last date,1-1-2000 00:00:00,pad_reversal.tgl terakhir yang diproses
+bphtb payment last id,0,bphtb.bphtb_payment.id terakhir yang diproses
+bphtb reversal last date,1-1-2000 00:00:00,bphtb.bphtb_reversal.tgl terakhir yang diproses

payment_report/scripts/pad.py

+import sys
+from datetime import timedelta
+import transaction
+from opensipkd.waktu import dmyhms
+from opensipkd.pad.models.default import (
+    IsoPayment,
+    IsoReversal,
+    Payment,
+    Invoice,
+    Pajak,
+    CustomerUsaha,
+    Customer,
+    )
+from opensipkd.pad.services.base import get_db_session
+from iso8583_web.models.meta import Base as BaseConf
+from opensipkd.iso8583.bjb.pad import Doc
+from ..models import (
+    Base,
+    Pad,
+    )
+from .common import (
+    get_iso,
+    get_keys,
+    App as BaseApp,
+    init_db as base_init_db,
+    )
+
+
+ERR_NOT_FOUND = 'Tgl {tgl_bayar} pad.pad_sspd.spt_id {invoice_id} tidak ada'
+
+
+class App(BaseApp):
+    iso_class = Doc
+    report_orm = Pad
+
+    def __init__(self, argv):
+        super().__init__(argv)
+        if not self.pid:
+            return
+        self.base_q_pay = self.prod_session.query(Payment)
+        self.base_q_inv = self.prod_session.query(Invoice, Pajak, Customer)
+        self.base_q_inv = self.base_q_inv.filter(
+                Invoice.pajak_id == Pajak.id,
+                Invoice.customer_usaha_id == CustomerUsaha.id,
+                CustomerUsaha.customer_id == Customer.id)
+
+    def get_db_session(self):  # Override
+        return get_db_session()
+
+    def get_pay(self, row):
+        tgl_bayar = row.tgl.date()
+        q_pay = self.base_q_pay.filter_by(spt_id=row.invoice_id).filter(
+                Payment.create_date >= tgl_bayar,
+                Payment.create_date < tgl_bayar + timedelta(1))
+        q_pay = q_pay.order_by(Payment.id.desc())
+        return q_pay.first()
+
+    def run_payment(self):
+        last = self.get_last_id('pad payment last id')
+        q_iso = self.prod_session.query(IsoPayment).filter(
+                IsoPayment.id > last.as_int())
+        for row in q_iso.order_by(IsoPayment.id):
+            iso = get_iso(row.iso_request, Doc, self.option.debug)
+            if self.get_report(iso):
+                continue
+            pay = self.get_pay(row)
+            tgl_bayar = row.tgl.date()
+            if not pay:
+                msg = ERR_NOT_FOUND.format(
+                        tgl_bayar=tgl_bayar, invoice_id=row.invoice_id)
+                self.log.error(msg)
+                continue
+            q_inv = self.base_q_inv.filter(Invoice.id == pay.spt_id)
+            inv, pajak, cust = q_inv.first()
+            d = get_keys(iso)
+            s_tgl = dmyhms(row.tgl)
+            self.log.info(
+                f'Tgl bayar {s_tgl}, Nomor bayar {d["nomor_bayar"]}, '
+                f'STAN {d["stan"]}, NTB {d["ntb"]}, Channel {d["channel"]}')
+            rpt = Pad(
+                    stan=d['stan'], ntb=d['ntb'], tgl=tgl_bayar,
+                    jam=row.tgl.time(), sspd_id=pay.id,
+                    nomor_bayar=iso.get_invoice_id().strip(),
+                    masa_pajak=pajak.masapajak, npwpd=cust.npwpd,
+                    nama_wp=cust.customernm, pokok=pay.jml_bayar-pay.denda,
+                    denda=pay.denda, jml_bayar=pay.jml_bayar,
+                    channel_id=iso.get_channel().strip(),
+                    channel_name=d['channel'])
+            last.nilai = str(row.id)
+            with transaction.manager:
+                self.rpt_session.add(rpt)
+                self.rpt_session.add(last)
+
+    def run_reversal(self):  # Override
+        super().run_reversal('pad reversal last date')
+
+
+def main(argv=sys.argv[1:]):
+    app = App(argv)
+    if app.pid:
+        app.run()
+
+
+def init_db(argv=sys.argv[1:]):
+    base_init_db(Base.metadata, BaseConf.metadata, argv)

setup.py

+import os
+from setuptools import (
+    setup,
+    find_packages,
+    )
+
+
+here = os.path.abspath(os.path.dirname(__file__))
+with open(os.path.join(here, 'README.rst')) as f:
+    README = f.read()
+with open(os.path.join(here, 'CHANGES.txt')) as f:
+    CHANGES = f.read()
+
+line = CHANGES.splitlines()[0]
+version = line.split()[0]
+
+requires = [
+    # 'opensipkd-iso8583-bjb @ '
+    # 'git+https://git.opensipkd.com/sugiana/opensipkd-iso8583-bjb.git',
+    ]
+
+
+setup(
+    name='payment-report',
+    version=version,
+    description='Pembuat tabel laporan',
+    long_description=README + '\n\n' + CHANGES,
+    author='Owo Sugiana',
+    author_email='sugiana@gmail.com',
+    license='PostgreSQL License',
+    packages=find_packages(),
+    include_package_data=True,
+    install_requires=requires,
+    zip_safe=False,
+    entry_points={
+        'console_scripts': [
+            'bphtb_report_init_db = payment_report.scripts.bphtb:init_db',
+            'pad_report_init_db = payment_report.scripts.pad:init_db',
+            'bphtb_report = payment_report.scripts.bphtb:main',
+            'pad_report = payment_report.scripts.pad:main',
+            ],
+        }
+    )