Commit f1458241 by Owo Sugiana

Tambah PBB dari API Cartenz

1 parent 14fd971e
3.1.6 2025-02-20
----------------
- Bug fixed Fortuna
- Tambah PBB dari API Cartenz
3.1.5 2024-10-23
----------------
- Tambah field kecamatan_op dan kelurahan_op pada table bphtb_report untuk
......
[main]
models = opensipkd.iso8583.bjb.pbb.models
# Tabel log_iso yang universal
# models = opensipkd.iso8583.bjb.models
service = opensipkd.iso8583.bjb.pbb.doc
db_url = postgresql://cartenz:pass@localhost/h2h_pbb
report_db_url = postgresql://pcpd:pass@localhost/pcpd
pid_file = /home/sugiana/tmp/pbb-iso.pid
log_file = /home/sugiana/log/pbb-iso.log
module = payment_report.log2iso.fortuna_pbb
report_db_url = postgresql://pcpd:pass@localhost/pbb
pid_file = /home/pcpd/tmp/pbb-log2iso.pid
[formatter_generic]
format = %(asctime)s %(levelname)s %(name)s %(message)s
[formatter_simple]
format = %(asctime)s %(levelname)s %(message)s
[handler_console]
class = StreamHandler
stream = sys.stdout
formatter = generic
[handler_file]
class = FileHandler
filename = /home/pcpd/logs/pbb-log2iso.log
formatter = simple
[logger_root]
handlers = console, file
level = INFO
......@@ -438,11 +438,15 @@ class App(BaseApp):
return q.first()
def get_kecamatan_nama(self, inv):
q = self.base_q_kecamatan.filter_by(kd_propinsi=inv.kd_propinsi, kd_dati2=inv.kd_dati2, kd_kecamatan=inv.kd_kecamatan)
q = self.base_q_kecamatan.filter_by(
kd_propinsi=inv.kd_propinsi, kd_dati2=inv.kd_dati2,
kd_kecamatan=inv.kd_kecamatan)
return q.first()
def get_kelurahan_nama(self, inv):
q = self.base_q_kelurahan.filter_by(kd_propinsi=inv.kd_propinsi, kd_dati2=inv.kd_dati2, kd_kecamatan=inv.kd_kecamatan, kd_kelurahan=inv.kd_kelurahan)
q = self.base_q_kelurahan.filter_by(
kd_propinsi=inv.kd_propinsi, kd_dati2=inv.kd_dati2,
kd_kecamatan=inv.kd_kecamatan, kd_kelurahan=inv.kd_kelurahan)
return q.first()
def create_data_sukabumi_kota(self, pay):
......@@ -520,14 +524,14 @@ class App(BaseApp):
perolehan = self.get_perolehan(inv)
kecamatan = self.get_kecamatan_nama(inv)
if not kecamatan:
msg = f'Field bphtb_sspd.kd_kecamatan {inv.kd_kecamatan} tidak ada di'\
'field ref_kecamatan.kd_kecamatan'
msg = f'Field bphtb_sspd.kd_kecamatan {inv.kd_kecamatan} '\
'tidak ada di field ref_kecamatan.kd_kecamatan'
raise InvalidSource(msg)
kelurahan = self.get_kelurahan_nama(inv)
if not kelurahan:
msg = f'Field bphtb_sspd.kd_kelurahan {inv.kd_kelurahan} tidak ada di'\
'field ref_kelurahan.kd_kelurahan'
msg = f'Field bphtb_sspd.kd_kelurahan {inv.kd_kelurahan} '\
'tidak ada di field ref_kelurahan.kd_kelurahan'
raise InvalidSource(msg)
invoice_id = FixLength(INVOICE_ID)
......
......@@ -71,7 +71,9 @@ BIT_PROFILE_NAMES = {
BANK_NAMES = {
'8': 'MANDIRI',
'14': 'BCA',
'700': 'PTPOS'}
'700': 'PTPOS',
'111': 'VA',
'112': 'QRIS'}
my_registry = dict()
......@@ -85,7 +87,11 @@ class VaInvoice(Base):
id = Column(Integer, primary_key=True)
va_type = Column(String(8))
product_code = Column(String(16))
invoice_no = Column(String(64))
invoice_no = Column(String(64), unique=True)
base_amount = Column(BigInteger)
fine = Column(BigInteger)
discount = Column(BigInteger)
customer_name = Column(String(64))
class VaPayment(Base):
......@@ -98,6 +104,17 @@ class VaPayment(Base):
customer_name = Column(String(64))
class VaTrxDapda(Base):
__tablename__ = 'trx_dapda'
id = Column(Integer, primary_key=True)
inv_code = Column(String(64), ForeignKey(VaInvoice.invoice_no))
payment_method = Column(String(32))
amount = Column(BigInteger)
fine = Column(BigInteger)
discount = Column(BigInteger)
total = Column(BigInteger)
def humanize_time(secs):
mins, secs = divmod(secs, 60)
hours, mins = divmod(mins, 60)
......@@ -217,6 +234,13 @@ class Conf(BaseConf):
pola += '.%f'
return datetime.strptime(self.nilai, pola)
def as_date(self):
pola = '%d-%m-%Y'
if self.nilai.find('.') > -1:
pola += '.%f'
t = datetime.strptime(self.nilai, pola)
return t.date()
def as_int(self):
return int(self.nilai)
......
from argparse import ArgumentParser
from ISO8583.ISOErrors import (
BitNotSet,
InvalidIso8583,
)
from sqlalchemy import func
from opensipkd.waktu import create_datetime
from opensipkd.iso8583.bjb.models import Log as BaseLog
from ..models import Common
from ..log_models import LogFile
from ..common import (
BaseApp,
InvalidSource,
)
def get_parser():
pars = ArgumentParser()
pars.add_argument('conf')
pars.add_argument('--update-from-id', type=int)
return pars
def get_option(argv):
pars = get_parser()
return pars.parse_args(argv)
class Log(BaseLog, Common):
pass
class LogApp(BaseApp):
report_orm = Log # Override
iso8583_class = None # Override, please
bit3_payment_code = None # Override, please
def get_session_for_save(self): # Override
return self.rpt_session
def prepare_query_filter(self): # Override
if 'id_awal' in self.conf:
self.id_awal = self.conf['id_awal']
else:
self.last = self.get_last_id(self.conf_name)
self.id_awal = self.last.as_int() + 1
self.id_akhir = self.conf.get('id_akhir')
def get_filter_query(self, q):
q = q.filter(LogFile.id >= self.id_awal)
if self.id_akhir:
return q.filter(LogFile.id <= self.id_akhir)
return q
def get_count(self): # Override
q = self.rpt_session.query(func.count())
q = self.get_filter_query(q)
return q.scalar()
def get_payment_query(self): # Override
q = self.rpt_session.query(LogFile)
q = self.get_filter_query(q)
return q.order_by(LogFile.id)
def get_prefix_log(self): # Override
return f'ID {self.log_id}'
def parse(self, line): # Override, please
pass
def create_data(self, log_file): # Override
self.log_id = log_file.id
d = self.parse(log_file.line)
waktu = create_datetime(
int(d['year']), int(d['month']), int(d['day']), int(d['hour']),
int(d['minute']), int(d['second']), int(int(d['milisecond'])/1000))
print([d['raw']])
iso = self.iso8583_class()
try:
iso.setIsoContent(d['raw'].encode('utf-8'))
except InvalidIso8583 as e:
raise InvalidSource(f'ID {log_file.id} tidak dipahami: {str(e)}')
except ValueError as e:
raise InvalidSource(f'ID {log_file.id} tidak dipahami: {str(e)}')
try:
if iso.getBit(3) != self.bit3_payment_code:
raise InvalidSource(f'ID {log_file.id} bukan payment')
except BitNotSet:
raise InvalidSource(f'ID {log_file.id} bit 3 tidak ada')
# Meski key pada dictionary bertipe integer namun Postgres menyimpannya
# bertipe string. Jadi string-kan dari sekarang agar tidak selalu mode
# UPDATE saat opsi --update-from-id
vals = iso.get_values()
bits = dict()
for bit, val in vals.items():
bits[str(bit)] = val
return dict(
id=log_file.id, mti=iso.getMTI(), created=waktu,
bits=bits)
def get_last_time(self): # Override
return str(self.log_id)
from opensipkd.string import FixLength
from opensipkd.iso8583.bjb.pbb import Doc
from opensipkd.iso8583.bjb.pbb.structure import (
PAYMENT_CODE,
INVOICE_PROFILE,
)
from . import LogApp
from .fortuna import RE_ISO
class App(LogApp):
conf_name = 'pbb log file last id'
iso8583_class = Doc
bit3_payment_code = PAYMENT_CODE
def parse(self, line): # Override
found = RE_ISO.search(line)
day, month, year, hour, minute, second, milisecond, raw = \
found.groups()
return dict(
day=day, month=month, year=year, hour=hour, minute=minute,
second=second, milisecond=milisecond, raw=raw)
def create_data(self, log_file): # Override
d = super().create_data(log_file)
raw = d['bits']['62']
if raw[:2] == '00':
raw = raw[2:]
profile = FixLength(INVOICE_PROFILE)
profile.set_raw(raw)
d['bits_data'] = {'62': profile.to_dict()}
return d
import sys
import re
from argparse import ArgumentParser
from time import time
from ISO8583.ISOErrors import (
......@@ -11,27 +12,18 @@ from opensipkd.waktu import create_datetime
from opensipkd.string import FixLength
from ..models import Common
from ..log_models import LogFile
from .common import (
from ..common import (
BaseApp,
my_registry,
append_csv,
create_session,
InvalidSource,
)
from .log2db_fortuna import RE_ISO
def get_parser():
pars = ArgumentParser()
pars.add_argument('conf')
pars.add_argument('--update-from-id', type=int)
pars.add_argument('--debug-sql', action='store_true')
return pars
def get_option(argv):
pars = get_parser()
return pars.parse_args(argv)
RE_ISO = r'([0-9]*)/([0-9]*)/([0-9]*) ([0-9]*):([0-9]*):([0-9]*)\.([0-9]*) '\
r'\[BJB\] Send : (.*)'
RE_ISO = re.compile(RE_ISO)
class App(BaseApp):
......@@ -95,7 +87,6 @@ class App(BaseApp):
raise InvalidSource(f'ID {log_file.id} bukan payment')
except BitNotSet:
raise InvalidSource(f'ID {log_file.id} bit 3 tidak ada')
waktu = iso.get_transaction_time()
d = dict(
id=log_file.id, mti=iso.getMTI(), created=waktu,
bits=iso.get_values())
......@@ -110,8 +101,11 @@ class App(BaseApp):
except BitNotSet:
pass
else:
raw = iso.getBit(62)
if raw[:2] == '00':
raw = raw[2:]
profile = FixLength(self.service.INVOICE_PROFILE)
profile.set_raw(iso.getBit(62))
profile.set_raw(raw)
d['bits_data'] = {62: profile.to_dict()}
return d
......@@ -138,5 +132,15 @@ class App(BaseApp):
def main(argv=sys.argv[1:]):
app = App(argv)
option = get_option(argv)
conf = ConfigParser()
conf.read(option.conf)
cf = dict(conf.items('main'))
if not make_pid_file(cf['pid_file']):
return
setup_logging(option.conf)
if option.update_from_id:
cf['from_id'] = option.update_from_id
module = get_module_object(conf.get('main', 'module'))
app = module.App(cf)
app.run()
from opensipkd.string import FixLength
from opensipkd.iso8583.bjb.pbb import Doc
from opensipkd.iso8583.bjb.pbb.structure import (
PAYMENT_CODE,
INVOICE_PROFILE,
)
from . import LogApp
from .fortuna import RE_ISO
class App(LogApp):
conf_name = 'pbb log file last id'
iso8583_class = Doc
bit3_payment_code = PAYMENT_CODE
def parse(self, line): # Override
found = RE_ISO.search(line)
day, month, year, hour, minute, second, milisecond, raw = \
found.groups()
return dict(
day=day, month=month, year=year, hour=hour, minute=minute,
second=second, milisecond=milisecond, raw=raw)
def create_data(self, log_file): # Override
d = super().create_data(log_file)
raw = d['bits']['62']
if raw[:2] == '00':
raw = raw[2:]
profile = FixLength(INVOICE_PROFILE)
profile.set_raw(raw)
d['bits_data'] = {'62': profile.to_dict()}
return d
......@@ -5,12 +5,13 @@ from sqlalchemy import (
Text,
)
from sqlalchemy.ext.declarative import declarative_base
from .models import Common
Base = declarative_base()
class LogFile(Base):
class LogFile(Base, Common):
__tablename__ = 'log_file'
id = Column(Integer, primary_key=True)
line_id = Column(String(64), nullable=False, unique=True)
......
......@@ -141,13 +141,13 @@ class Bphtb(Base, Common):
channel_nama = Column(String(32), nullable=False)
# bphtb.bphtb_reversal.tgl
tgl_batal = Column(DateTime(timezone=True))
status_pembayaran = Column(Integer)
# pbb.ref_kecamatan.nm_kecamatan
kecamatan_op = Column(String(64))
# pbb.ref_kelurahan.nm_kelurahan
kelurahan_op = Column(String(64))
class Pbb(Base, Common):
__tablename__ = 'pbb_report'
id = Column(Integer, primary_key=True)
......
from datetime import (
date,
datetime,
timedelta,
)
from time import time
import requests
from logging import getLogger
from opensipkd.string import FixLength
from sismiop.services.base import NOP
import transaction
from ..models import Pbb
from ..common import BaseApp
from ..tools import (
plain_values,
update,
)
one_day = timedelta(1)
def create_data(d):
nop = FixLength(NOP)
nop.set_raw(d['nop'])
tgl = ' '.join([d['tanggal_tx'], d['jam_tx']])
tgl = datetime.strptime(tgl, '%Y-%m-%d %H:%M:%S')
if d['channel_pembayaran'].find('BANK JABAR') > -1:
bank_id = '110'
else:
bank_id = None
return dict(
kd_propinsi=nop['Propinsi'], kd_dati2=nop['Kabupaten'],
kd_kecamatan=nop['Kecamatan'], kd_kelurahan=nop['Kelurahan'],
kd_blok=nop['Blok'], no_urut=nop['Urut'],
kd_jns_op=nop['Jenis'], thn_pajak_sppt=d['tahun'],
pembayaran_sppt_ke=1,
jml_sppt_yg_dibayar=int(d['total']), denda_sppt=int(d['denda']),
discount=int(d['potongan']), tgl_pembayaran_sppt=tgl.date(),
tgl_rekam_byr_sppt=tgl, nm_wp_sppt=d['nama_wp'], channel_kode='0000',
channel_nama='LAINNYA', bank_id=bank_id,
pbb_yg_harus_dibayar_sppt=int(d['pokok_pajak']))
class App(BaseApp):
report_orm = Pbb
conf_name = 'pbb api last date'
def get_report(self, d):
session = self.get_session_for_save()
q = session.query(self.report_orm).filter_by(
kd_propinsi=d['kd_propinsi'], kd_dati2=d['kd_dati2'],
kd_kecamatan=d['kd_kecamatan'], kd_kelurahan=d['kd_kelurahan'],
kd_blok=d['kd_blok'], no_urut=d['no_urut'],
kd_jns_op=d['kd_jns_op'], thn_pajak_sppt=d['thn_pajak_sppt'],
pembayaran_sppt_ke=d['pembayaran_sppt_ke'])
return q.first()
def do_sync(self, tgl):
d = dict(tanggal=tgl.strftime('%Y-%m-%d'))
r = requests.post(self.conf['url'], json=d)
if r.status_code != 200:
print('HTTP Error', r.status_code)
print(r.text)
return
log = getLogger('do_sync()')
d = r.json()
no = 0
self.count = len(d['transaksi'])
self.last_pay = None
for trx in d['transaksi']:
self.invoice_id = ''.join([trx['nop'], trx['tahun']])
data = create_data(trx)
no += 1
source = create_data(trx)
d = plain_values(source)
rpt = self.get_report(data)
if rpt:
target = rpt.to_dict()
target_update, log_msg = update(source, target)
if target_update:
s = ', '.join(log_msg)
msg = f'UPDATE {d} change {s}'
rpt.from_dict(target_update)
else:
msg = f'ALREADY SAME {d}'
rpt = None
if self.count == 1 and self.last: # Hemat log
print(msg)
print('Log yang sama, abaikan.')
return
else:
msg = f'INSERT {d}'
rpt = self.report_orm(**source)
e = self.get_estimate(no)
prefix = self.get_prefix_log()
log.info(f'#{no}/{self.count} {prefix} {msg}, estimate {e}')
if rpt:
session = self.get_session_for_save()
with transaction.manager:
session.add(rpt)
self.last_pay = source
return self.count
def get_last_time(self): # Override
return self.last_pay['tgl_pembayaran_sppt'].strftime('%d-%m-%Y')
def prepare_query_filter(self):
if 'tgl_awal' in self.conf:
self.last = None
self.tgl_awal = self.conf['tgl_awal']
else:
self.last = self.get_last_id(self.conf_name)
self.tgl_awal = self.last.as_date()
if 'tgl_akhir' in self.conf:
self.tgl_akhir = self.conf['tgl_akhir']
else:
self.tgl_akhir = date.today()
def run(self): # dipanggil sync.py
self.prepare_query_filter()
tgl = self.tgl_awal
while True:
tgl += one_day
self.start_time = time()
self.do_sync(tgl)
if self.last_pay and self.last:
self.update_last()
if tgl > self.tgl_akhir:
break
......@@ -33,9 +33,11 @@ class App(BaseApp):
VaInvoice.va_type,
VaPayment.id,
VaPayment.rrn,
VaPayment.customer_name,
VaInvoice.customer_name,
VaPayment.transaction_date,
VaPayment.transaction_amount)
VaPayment.transaction_amount,
VaInvoice.fine,
VaInvoice.discount)
q = self.filter_query(q)
return q.order_by(VaPayment.id)
......@@ -58,6 +60,11 @@ class App(BaseApp):
inv_id.set_raw(pay.invoice_no)
tgl = datetime.strptime(pay.transaction_date, '%Y-%m-%d %H:%M:%S')
rpt_id = - pay.id
denda = pay.fine or 0
diskon = pay.discount or 0
diskon_denda = denda-diskon
bayar = pay.transaction_amount or 0
pokok = bayar-diskon_denda
return dict(
id=rpt_id,
kd_propinsi=inv_id['Propinsi'], kd_dati2=inv_id['Kabupaten'],
......@@ -65,12 +72,12 @@ class App(BaseApp):
kd_blok=inv_id['Blok'], no_urut=inv_id['Urut'],
kd_jns_op=inv_id['Jenis'], thn_pajak_sppt=inv_id['Tahun'],
pembayaran_sppt_ke=1, ntb=pay.rrn,
jml_sppt_yg_dibayar=pay.transaction_amount, denda_sppt=0,
discount=0, tgl_pembayaran_sppt=tgl.date(),
jml_sppt_yg_dibayar=bayar, denda_sppt=diskon_denda,
discount=diskon, tgl_pembayaran_sppt=tgl.date(),
tgl_rekam_byr_sppt=tgl,
nm_wp_sppt=pay.customer_name, channel_kode='0000',
channel_nama=channel_nama, bank_id='110',
pbb_yg_harus_dibayar_sppt=pay.transaction_amount)
pbb_yg_harus_dibayar_sppt=pokok)
def get_report(self, pay): # Override
inv_id = FixLength(INVOICE_ID)
......
......@@ -4,3 +4,4 @@ webr payment last date,1-1-2000 00:00:00,webr.ar_payment.created terakhir yang d
bphtb payment last date,01-01-2000 00:00:00,bphtb.bphtb_bank.tanggal terakhir yang diproses
pbb payment last date,1-1-2000 00:00:00,pembayaran_sppt.tgl_rekam_byr_sppt terakhir yang diproses
pbb va payment last date,1-1-2000 00:00:00,bjb_va_payment.transaction_date terakhir yang diproses
pbb api last date,31-12-2024 00:00:00,tanggal terakhir yang diproses
......@@ -13,13 +13,9 @@ from sqlalchemy.exc import (
OperationalError,
)
from ..log_models import LogFile
from .tools import BacaFile
from .common import make_pid_file
RE_ISO = r'([0-9]*)/([0-9]*)/([0-9]*) ([0-9]*):([0-9]*):([0-9]*)\.([0-9]*) '\
r'\[BJB\] Send : (.*)'
RE_ISO = re.compile(RE_ISO)
from ..tools import BacaFile
from ..common import make_pid_file
from ..common_fortuna import RE_ISO
def read_log(log_file, db_url):
......
......@@ -19,13 +19,6 @@ from .common import (
)
PATTERN = r'^([\d]*)-([\d]*)-([\d]*) ([\d]*):([\d]*):([\d]*)\.([\d]*) '\
r'(.*) \[SENT RAW BUFFER\] : (.*)'
REGEX = re.compile(PATTERN)
Base = declarative_base()
def get_parser():
pars = ArgumentParser()
pars.add_argument('conf')
......@@ -146,5 +139,15 @@ class App(BaseApp):
def main(argv=sys.argv[1:]):
app = App(argv)
option = get_option(argv)
conf = ConfigParser()
conf.read(option.conf)
cf = dict(conf.items('main'))
if not make_pid_file(cf['pid_file']):
return
setup_logging(option.conf)
if option.update_from_id:
cf['from_id'] = option.update_from_id
module = get_module_object(conf.get('main', 'module'))
app = module.App(cf)
app.run()
......@@ -3,7 +3,7 @@ from configparser import ConfigParser
from sqlalchemy import create_engine
from opensipkd.views.models import Conf
from ..log_models import Base
from .common import (
from ..common import (
my_registry,
append_csv,
create_session,
......
......@@ -9,7 +9,7 @@ from .sync import make_pid_file
def get_parser():
pars = ArgumentParser()
pars.add_argument('conf')
pars.add_argument('--update-from-id', type=int)
pars.add_argument('--update-from-id')
return pars
......@@ -27,7 +27,12 @@ def main(argv=sys.argv[1:]):
return
setup_logging(option.conf)
if option.update_from_id:
cf['start_id'] = option.update_from_id
try:
cf['id_awal'] = int(option.update_from_id)
except ValueError:
t = option.update_from_id.split(',')
cf['id_awal'] = int(t[0])
cf['id_akhir'] = int(t[1])
module = get_module_object(conf.get('main', 'module'))
app = module.App(cf)
app.run()
[main]
module = payment_report.pbb.cartenz_api
url = https://pbb.tangerangkab.go.id/backend-api/revenue/tax/property/integrasi/sipari/penerimaan-harian
report_db_url = postgresql://user:pass@localhost/db
pid_file = /home/pcpd/tmp/pbb-api-report.pid
[formatter_generic]
format = %(asctime)s %(levelname)s %(name)s %(message)s
[formatter_simple]
format = %(asctime)s %(levelname)s %(message)s
[handler_console]
class = StreamHandler
stream = sys.stdout
formatter = generic
[handler_file]
class = FileHandler
filename = /home/pcpd/logs/pbb-api-report.log
formatter = simple
[logger_root]
handlers = console, file
level = INFO
......@@ -40,7 +40,6 @@ setup(
'pbb_log = payment_report.scripts.pbb_log:main',
'log2iso_init_db = payment_report.scripts.log2iso_init:main',
'log2iso = payment_report.scripts.log2iso:main',
'log2iso_fortuna = payment_report.scripts.log2iso_fortuna:main',
'log2db = payment_report.scripts.log2db:main',
'log2db_ciamis = payment_report.scripts.log2db_ciamis:main',
'log2db_fortuna = payment_report.scripts.log2db_fortuna:main',
......
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!