Tambah PBB Ciamis

1 parent 10f8ac97
2.10 2023-03-07
---------------
- Tambah PBB Ciamis
2.9 2023-03-01 2.9 2023-03-01
-------------- --------------
- Tambah BPHTB Kabupaten Serang - Tambah BPHTB Kabupaten Serang
......
[main] [main]
models = opensipkd.iso8583.bjb.pbb.models models = opensipkd.iso8583.bjb.pbb.models
# Tabel log_iso yang universal
# models = opensipkd.iso8583.bjb.models
service = opensipkd.iso8583.bjb.pbb.doc service = opensipkd.iso8583.bjb.pbb.doc
db_url = postgresql://cartenz:pass@localhost/h2h_pbb db_url = postgresql://cartenz:pass@localhost/h2h_pbb
report_db_url = postgresql://pcpd:pass@localhost/pcpd report_db_url = postgresql://pcpd:pass@localhost/pcpd
......
import requests
from sqlalchemy import func
import transaction
from opensipkd.waktu import dmyhms
from opensipkd.string import FixLength
from sismiop.services.base import INVOICE_ID
from opensipkd.iso8583.bjb.pbb.structure import (
PAYMENT_CODE,
INVOICE_PROFILE,
)
from opensipkd.iso8583.bjb.models import Log
from payment_report.scripts.tools import (
plain_values,
update,
)
from payment_report.scripts.common import (
InvalidSource,
one_day,
get_channel_name_by_row,
row_limit,
)
class Handler:
def __init__(self, parent):
self.parent = parent
def filter_tgl(self, q):
q = q.filter(
Log.created >= self.parent.tgl_awal,
Log.created < self.parent.tgl_akhir + one_day,
Log.bits.op('->>')('3') == PAYMENT_CODE,
Log.bits.op('->>')('39') == '00')
return q.filter_by(mti='0210')
def get_payment_query(self):
q = self.parent.h2h_session.query(Log)
return self.filter_tgl(q).order_by(Log.id)
def get_count(self): # dipanggil parent
q = self.parent.h2h_session.query(func.count(Log.id))
q = self.filter_tgl(q)
return q.scalar()
def get_last_time(self): # dipanggil parent
return dmyhms(self.parent.last_pay.created)
def get_report(self, pay):
session = self.parent.get_session_for_save()
q = session.query(self.parent.report_orm).filter_by(id=pay.id)
return q.first()
def create_data(self, pay):
channel_nama = get_channel_name_by_row(pay.bits)
channel_kode = pay.bits['18'] or '0000'
profile = pay.bits_data['62']
inv_id = FixLength(INVOICE_ID)
inv_id.set_raw(pay.bits['61'])
return dict(
id=pay.id, kd_propinsi=inv_id['Propinsi'],
kd_dati2=inv_id['Kabupaten'], kd_kecamatan=inv_id['Kecamatan'],
kd_kelurahan=inv_id['Kelurahan'], kd_blok=inv_id['Blok'],
no_urut=inv_id['Urut'], kd_jns_op=inv_id['Jenis'],
thn_pajak_sppt=inv_id['Tahun'], pembayaran_sppt_ke=1,
stan=pay.bits['11'], ntb=pay.bits['48'],
jml_sppt_yg_dibayar=float(pay.bits['4']),
denda_sppt=float(profile['Denda']),
discount=float(profile['Discount']),
tgl_pembayaran_sppt=pay.created.date(),
tgl_rekam_byr_sppt=pay.created, nm_wp_sppt=profile['Nama'],
channel_kode=channel_kode, channel_nama=channel_nama,
bank_id=pay.bits['32'], user_id=pay.bits['107'],
pbb_yg_harus_dibayar_sppt=float(profile['Tagihan']))
def update_from_date(self):
q = self.get_payment_query()
no = self.parent.offset
found = False
for pay in q.offset(self.parent.offset).limit(row_limit):
found = True
self.parent.invoice_id = pay.bits['61'].strip()
no += 1
try:
source = self.create_data(pay)
d = plain_values(source)
rpt = self.get_report(pay)
if rpt:
target = rpt.to_dict()
target_update, log_msg = update(source, target)
if target_update:
s = ', '.join(log_msg)
msg = f'UPDATE {d} change {s}'
rpt.from_dict(target_update)
else:
msg = f'ALREADY SAME {d}'
rpt = None
if self.parent.count == 1 and self.parent.last:
print(msg)
print('Log yang sama, abaikan.')
return
else:
msg = f'INSERT {d}'
rpt = self.parent.report_orm(**source)
msg = f'{self.parent.get_prefix_log()} {msg}'
log_method = self.parent.log.info
except InvalidSource as e:
msg = str(e)
log_method = self.parent.log.warning
rpt = None
e = self.parent.get_estimate(no)
log_method(f'#{no}/{self.parent.count} {msg}, estimate {e}')
if rpt:
session = self.parent.get_session_for_save()
with transaction.manager:
session.add(rpt)
self.parent.last_pay = pay
self.parent.offset += row_limit
return found
...@@ -182,11 +182,16 @@ def get_channel_info_by_iso(iso): ...@@ -182,11 +182,16 @@ def get_channel_info_by_iso(iso):
def get_channel_name_by_row(row): def get_channel_name_by_row(row):
if isinstance(row, dict):
return get_channel_name_by_dict(row)
return get_channel_name( return get_channel_name(
row.bit_018, row.bit_032, row.bit_041, row.bit_042, row.bit_043) row.bit_018, row.bit_032, row.bit_041, row.bit_042, row.bit_043)
def get_channel_name_by_dict(d): def get_channel_name_by_dict(d):
if '18' in d:
return get_channel_name(
d['18'], d['32'], d['41'], d['42'], d['43'])
return get_channel_name( return get_channel_name(
d['bit_018'], d['bit_032'], d.get('bit_041'), d.get('bit_042'), d['bit_018'], d['bit_032'], d.get('bit_041'), d.get('bit_042'),
d.get('bit_043')) d.get('bit_043'))
......
import sys
import os
import re
from argparse import ArgumentParser
from configparser import ConfigParser
from hashlib import sha256
from time import sleep
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import (
IntegrityError,
OperationalError,
)
from ..log_models import LogFile
from .tools import BacaFile
RE_CONNECTED = re.compile('connected$')
def read_log(log_file, db_url):
def save_log(line):
line = line.rstrip().rstrip('\x03')
print([line])
found = RE_CONNECTED.search(line)
if found:
return
print([line])
line_id = sha256(line.encode('utf-8')).hexdigest()
while True:
row = LogFile(line_id=line_id, line=line)
try:
db_session.add(row)
db_session.flush()
db_session.commit()
return
except IntegrityError:
db_session.rollback()
return
except OperationalError as e:
db_session.rollback()
print(e)
print('Tunggu, mungkin sedang maintenance ...')
sleep(10)
engine = create_engine(db_url)
factory = sessionmaker(bind=engine)
db_session = factory()
with BacaFile(log_file) as f:
while True:
line = f.readline()
if not line:
break
try:
save_log(line.decode('utf-8'))
except UnicodeDecodeError:
continue
def main(argv=sys.argv[1:]):
pars = ArgumentParser()
pars.add_argument('conf')
pars.add_argument('--log-file', required=True)
option = pars.parse_args(argv)
conf = ConfigParser()
conf.read(option.conf)
db_url = conf.get('main', 'db_url')
read_log(option.log_file, db_url)
...@@ -93,7 +93,7 @@ class App(BaseApp): ...@@ -93,7 +93,7 @@ class App(BaseApp):
try: try:
iso.setIsoContent(raw[4:].encode('utf-8')) iso.setIsoContent(raw[4:].encode('utf-8'))
except ValueError as e: except ValueError as e:
raise InvalidSource('ID {log_file.id} tidak dipahami: {str(e)}') raise InvalidSource(f'ID {log_file.id} tidak dipahami: {str(e)}')
try: try:
if iso.getBit(3) != self.service.PAYMENT_CODE: if iso.getBit(3) != self.service.PAYMENT_CODE:
raise InvalidSource(f'ID {log_file.id} bukan payment') raise InvalidSource(f'ID {log_file.id} bukan payment')
......
import sys
import re
from argparse import ArgumentParser
from time import time
from ISO8583.ISOErrors import BitNotSet
from sqlalchemy import func
from sqlalchemy.ext.declarative import declarative_base
from zope.sqlalchemy import register
from opensipkd.waktu import create_datetime
from opensipkd.string import FixLength
from ..models import Common
from ..log_models import LogFile
from .common import (
BaseApp,
my_registry,
append_csv,
create_session,
InvalidSource,
)
PATTERN = r'^([\d]*)-([\d]*)-([\d]*) ([\d]*):([\d]*):([\d]*)\.([\d]*) '\
r'(.*) \[SENT RAW BUFFER\] : (.*)'
REGEX = re.compile(PATTERN)
Base = declarative_base()
def get_parser():
pars = ArgumentParser()
pars.add_argument('conf')
pars.add_argument('--update-from-id', type=int)
pars.add_argument('--debug-sql', action='store_true')
return pars
def get_option(argv):
pars = get_parser()
return pars.parse_args(argv)
class App(BaseApp):
def __init__(self, argv):
super().__init__(argv)
if not self.pid:
return
class Log(self.models.Log, Common):
pass
self.report_orm = Log
if self.conf['models'].find('bphtb') > -1:
self.conf_name = 'bphtb log file last id'
self.pajak = 'bphtb'
else:
self.conf_name = 'pbb log file last id'
self.pajak = 'pbb'
register(self.prod_session)
def get_option(self, argv): # Override
return get_option(argv)
def get_session_for_save(self): # Override
return self.prod_session
def get_filter_query(self, q):
return q.filter(LogFile.id > self.last_id)
def get_count(self): # Override
q = self.prod_session.query(func.count())
q = self.get_filter_query(q)
return q.scalar()
def get_payment_query(self): # Override
q = self.prod_session.query(LogFile)
q = self.get_filter_query(q)
return q.order_by(LogFile.id)
def get_prefix_log(self): # Override
return f'ID {self.log_id}'
def create_data(self, log_file): # Override
self.log_id = log_file.id
raw = log_file.line[15:]
iso = self.service.Doc()
try:
iso.setIsoContent(raw.encode('utf-8'))
except ValueError as e:
raise InvalidSource(f'ID {log_file.id} tidak dipahami: {str(e)}')
try:
if iso.getBit(3) != self.service.PAYMENT_CODE:
raise InvalidSource(f'ID {log_file.id} bukan payment')
except BitNotSet:
raise InvalidSource(f'ID {log_file.id} bit 3 tidak ada')
waktu = iso.get_transaction_time()
d = dict(
id=log_file.id, mti=iso.getMTI(), created=waktu,
bits=iso.get_values())
if self.pajak == 'bphtb':
try:
profile = FixLength(self.service.INVOICE_PROFILE)
profile.set_raw(iso.getBit(47))
d['bit_047_data'] = profile.to_dict()
profile = FixLength(self.service.INVOICE_PROFILE2)
profile.set_raw(iso.getBit(48))
d['bit_048_data'] = profile.to_dict()
except BitNotSet:
pass
else:
profile = FixLength(self.service.INVOICE_PROFILE)
profile.set_raw(iso.getBit(62))
d['bits_data'] = {62: profile.to_dict()}
return d
def get_last_time(self): # Override
return str(self.last_pay.id)
def run_payment(self): # Override
self.last_pay = None
self.offset = 0
if self.option.update_from_id is None:
self.last = self.get_last_id(self.conf_name)
self.last_id = self.last.as_int()
else:
self.last = None
self.last_id = self.option.update_from_id - 1
self.count = self.get_count()
self.start_time = time()
while True:
found = self.update_from_date()
if not found:
break
if self.last_pay and self.last:
self.update_last()
def main(argv=sys.argv[1:]):
app = App(argv)
app.run()
...@@ -28,6 +28,7 @@ from opensipkd.iso8583.bjb.scripts.common import get_module_object ...@@ -28,6 +28,7 @@ from opensipkd.iso8583.bjb.scripts.common import get_module_object
from opensipkd.iso8583.bjb.pbb.default import Doc from opensipkd.iso8583.bjb.pbb.default import Doc
from opensipkd.iso8583.bjb.pbb.structure import PAYMENT_CODE from opensipkd.iso8583.bjb.pbb.structure import PAYMENT_CODE
from opensipkd.iso8583.bjb.pbb.models import Log from opensipkd.iso8583.bjb.pbb.models import Log
from opensipkd.iso8583.bjb.models import Log as UniversalLog
from ..models import ( from ..models import (
Pbb, Pbb,
Base, Base,
...@@ -116,14 +117,15 @@ class App(BaseApp): ...@@ -116,14 +117,15 @@ class App(BaseApp):
self.base_q_iso = self.prod_session.query( self.base_q_iso = self.prod_session.query(
self.iso_payment_orm, self.iso_inquiry_orm).filter( self.iso_payment_orm, self.iso_inquiry_orm).filter(
self.iso_payment_orm.inquiry_id == self.iso_inquiry_orm.id) self.iso_payment_orm.inquiry_id == self.iso_inquiry_orm.id)
self.models = get_module_object(self.conf['models']) if 'models' in self.conf:
self.Psppt = self.models.PembayaranSppt self.models = get_module_object(self.conf['models'])
self.Sppt = self.models.Sppt self.Psppt = self.models.PembayaranSppt
self.base_q_sppt = self.prod_session.query(self.Sppt) self.Sppt = self.models.Sppt
self.nip_pospbb = self.conf.get('nip_pospbb') self.base_q_sppt = self.prod_session.query(self.Sppt)
self.kd_tp_kasda = self.conf.get('kd_tp_kasda') self.nip_pospbb = self.conf.get('nip_pospbb')
self.kd_tp_va = self.conf.get('kd_tp_va') self.kd_tp_kasda = self.conf.get('kd_tp_kasda')
self.kd_tp_qris = self.conf.get('kd_tp_qris') self.kd_tp_va = self.conf.get('kd_tp_va')
self.kd_tp_qris = self.conf.get('kd_tp_qris')
def set_other_db(self): def set_other_db(self):
if 'h2h_db_url' in self.conf: if 'h2h_db_url' in self.conf:
......
import sys
from sqlalchemy import (
Column,
Integer,
String,
UniqueConstraint,
)
import transaction
from opensipkd.string import (
FixLength,
DateTimeVar,
)
from opensipkd.waktu import (
dmyhms,
create_datetime,
date_from_str,
)
from sismiop.services.base import get_id
from ..models import (
Pbb as Report,
Base,
)
from .common import (
BaseApp,
get_channel_name_by_dict,
get_module_object,
)
ERR_NOT_FOUND = 'SPPT ID {invoice_id} pembayaran ke {ke} '\
'tidak ada di pembayaran_sppt'
class Log(Base):
__tablename__ = 'pbb_payment'
id = Column(Integer, primary_key=True)
propinsi = Column(String(2), nullable=False)
kabupaten = Column(String(2), nullable=False)
kecamatan = Column(String(3), nullable=False)
kelurahan = Column(String(3), nullable=False)
blok = Column(String(3), nullable=False)
urut = Column(String(4), nullable=False)
jenis = Column(String(1), nullable=False)
tahun = Column(Integer, nullable=False)
ke = Column(Integer, nullable=False)
kd_kanwil_bank = Column(String(2), nullable=False)
kd_kppbb_bank = Column(String(2), nullable=False)
kd_bank_tunggal = Column(String(2), nullable=False)
kd_bank_persepsi = Column(String(2), nullable=False)
kd_tp = Column(String(2), nullable=False)
channel = Column(String(4))
ntb = Column(String(64))
ntp = Column(String(64))
bank = Column(Integer)
iso_request = Column(String(2048), nullable=False)
__table_args__ = (
UniqueConstraint('propinsi', 'kabupaten', 'kecamatan', 'kelurahan',
'blok', 'urut', 'jenis', 'tahun', 'ke'),
dict(schema='public'))
def get_keys(d):
r = dict()
r['channel'] = get_channel_name_by_dict(d)
r.update(dict(
nomor_bayar=d['bit_061'],
stan=d['bit_011'],
ntb=d['bit_037']))
return r
def get_nama_wp(d):
return d['bit_062']['name']
def get_tgl_bayar(d, year):
raw = d['bit_007']
t = DateTimeVar()
t.set_raw(raw)
return create_datetime(
year, int(t['month']), int(t['day']), int(t['hour']),
int(t['minute']), int(t['second']))
def error(s):
print(s)
sys.exit()
class App(BaseApp):
report_orm = Report
def __init__(self, argv):
super().__init__(argv)
if not self.pid:
return
factory = self.get_factory('pbb_db_url')
self.pbb_session = factory()
self.base_q_log = self.prod_session.query(Log)
self.models = get_module_object(self.conf['models'])
self.PembayaranSppt = self.models.PembayaranSppt
self.Sppt = self.models.Sppt
self.base_q_psppt = self.pbb_session.query(self.PembayaranSppt)
self.base_q_psppt = self.base_q_psppt.filter_by(
kd_kanwil=self.conf['kd_kanwil'],
kd_kantor=self.conf['kd_kantor'], kd_tp=self.conf['kd_tp'])
self.base_q_sppt = self.pbb_session.query(self.Sppt)
def get_report(self, d): # Override
return super().get_report(d['bit_011'], d['bit_037'])
def get_psppt(self, pay):
q = self.base_q_psppt.filter_by(
kd_propinsi=pay.propinsi,
kd_dati2=pay.kabupaten,
kd_kecamatan=pay.kecamatan,
kd_kelurahan=pay.kelurahan,
kd_blok=pay.blok,
no_urut=pay.urut,
kd_jns_op=pay.jenis,
thn_pajak_sppt=str(pay.tahun),
pembayaran_sppt_ke=pay.ke)
return q.first()
def get_sppt(self, psppt):
q = self.base_q_sppt.filter_by(
kd_propinsi=psppt.kd_propinsi,
kd_dati2=psppt.kd_dati2,
kd_kecamatan=psppt.kd_kecamatan,
kd_kelurahan=psppt.kd_kelurahan,
kd_blok=psppt.kd_blok,
no_urut=psppt.no_urut,
kd_jns_op=psppt.kd_jns_op,
thn_pajak_sppt=psppt.thn_pajak_sppt)
return q.first()
def __get_query_log(self, last_id):
return self.base_q_log.filter(Log.id > last_id).order_by(Log.id)
def __log_not_found(self, row, d):
msg = ERR_NOT_FOUND.format(invoice_id=d['nomor_bayar'], ke=row.ke)
self.log.error(msg)
def __log_progress(self, s_tgl, d):
self.log.info(f'Tgl bayar {s_tgl}, {d}')
def __create_data(self, iso, d, tgl_bayar, psppt):
sppt = self.get_sppt(psppt)
nama_wp = get_nama_wp(iso)
discount = hasattr(psppt, 'discount') and psppt.discount or None
return dict(
stan=d['stan'], ntb=d['ntb'],
kd_propinsi=psppt.kd_propinsi, kd_dati2=psppt.kd_dati2,
kd_kecamatan=psppt.kd_kecamatan, kd_kelurahan=psppt.kd_kelurahan,
kd_blok=psppt.kd_blok, no_urut=psppt.no_urut,
kd_jns_op=psppt.kd_jns_op, thn_pajak_sppt=psppt.thn_pajak_sppt,
pembayaran_sppt_ke=psppt.pembayaran_sppt_ke,
jml_sppt_yg_dibayar=iso['bit_004'],
denda_sppt=psppt.denda_sppt, tgl_pembayaran_sppt=tgl_bayar.date(),
tgl_rekam_byr_sppt=tgl_bayar, nm_wp_sppt=nama_wp,
channel_kode=iso['bit_032'].strip(), channel_nama=d['channel'],
pbb_yg_harus_dibayar_sppt=sppt.pbb_yg_harus_dibayar_sppt,
discount=discount, bank_id=iso['bit_032'], tgl_inquiry=tgl_bayar)
def __run_payment(self):
last = self.get_last_id('pbb json payment last id')
q_log = self.__get_query_log(last.as_int())
found = False
for row in q_log.limit(1000):
iso = eval(row.iso_request)
if self.get_report(iso):
continue
d = get_keys(iso)
psppt = self.get_psppt(row)
if not psppt:
self.__log_not_found(row, d)
continue
tgl_bayar = get_tgl_bayar(iso, psppt.tgl_rekam_byr_sppt.year)
s_tgl = dmyhms(tgl_bayar)
self.__log_progress(s_tgl, d)
d = self.__create_data(iso, d, tgl_bayar, psppt)
rpt = Report(**d)
last.nilai = str(row.id)
with transaction.manager:
self.rpt_session.add(rpt)
self.rpt_session.add(last)
found = True
return found
def __update_from_id(self):
q_log = self.__get_query_log(self.last_id)
found = False
for row_pay in q_log.limit(1000):
iso = eval(row_pay.iso_request)
d = get_keys(iso)
psppt = self.get_psppt(row_pay)
if not psppt:
self.__log_not_found(row_pay, d)
continue
tgl_bayar = get_tgl_bayar(iso, psppt.tgl_rekam_byr_sppt.year)
s_tgl = dmyhms(tgl_bayar)
self.__log_progress(s_tgl, d)
d = self.__create_data(iso, d, tgl_bayar, psppt)
rpt = self.get_report(iso)
if not rpt:
rpt = Pbb()
rpt.from_dict(d)
with transaction.manager:
self.rpt_session.add(rpt)
found = True
self.last_id = row_pay.id
return found
def run_payment(self): # Override
if self.option.update_from_id:
self.last_id = self.option.update_from_id
func = self.__update_from_id
elif self.option.update_from_date:
tgl = date_from_str(self.option.update_from_date)
Psppt = self.PembayaranSppt
q = self.base_q_psppt.filter(Psppt.tgl_rekam_byr_sppt >= tgl)
row_psppt = q.order_by(Psppt.tgl_rekam_byr_sppt).first()
if not row_psppt:
error('Kosong')
invoice_id = get_id(row_psppt)
print(f'DEBUG Invoice ID {invoice_id}')
q = self.base_q_log.filter_by(
propinsi=row_psppt.kd_propinsi,
kabupaten=row_psppt.kd_dati2,
kecamatan=row_psppt.kd_kecamatan,
kelurahan=row_psppt.kd_kelurahan,
blok=row_psppt.kd_blok,
urut=row_psppt.no_urut,
jenis=row_psppt.kd_jns_op,
tahun=row_psppt.thn_pajak_sppt,
ke=row_psppt.pembayaran_sppt_ke)
row_log = q.order_by(Log.id).first()
self.last_id = row_log.id - 1
func = self.__update_from_id
else:
func = self.__run_payment
while True:
found = func()
if not found:
break
def main(argv=sys.argv[1:]):
app = App(argv)
app.run()
...@@ -97,7 +97,8 @@ class App(BaseApp): ...@@ -97,7 +97,8 @@ class App(BaseApp):
channel_name = 'VA' channel_name = 'VA'
else: else:
channel_id = '0000' channel_id = '0000'
channel_name = self.get_va_channel(pay.created.date()) or 'MANUAL' tgl = pay.created.date()
channel_name = self.get_va_channel(tgl) or 'MANUAL'
stan = iso and iso.bit_011 or None stan = iso and iso.bit_011 or None
ntb = pay.ntb and pay.ntb.strip() or None ntb = pay.ntb and pay.ntb.strip() or None
return dict( return dict(
......
...@@ -42,7 +42,9 @@ setup( ...@@ -42,7 +42,9 @@ setup(
'pbb_log = payment_report.scripts.pbb_log:main', 'pbb_log = payment_report.scripts.pbb_log:main',
'log2iso_init_db = payment_report.scripts.log2iso_init:main', 'log2iso_init_db = payment_report.scripts.log2iso_init:main',
'log2iso = payment_report.scripts.log2iso:main', 'log2iso = payment_report.scripts.log2iso:main',
'log2iso_ciamis = payment_report.scripts.log2iso_ciamis:main',
'log2db = payment_report.scripts.log2db:main', 'log2db = payment_report.scripts.log2db:main',
'log2db_ciamis = payment_report.scripts.log2db_ciamis:main',
], ],
} }
) )
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!