Skip to content

Owo Sugiana / payment-report

Go to a project
Commit e856dc2b by Owo Sugiana
Options

Tambah By ID selain By Date

1 parent e6e71033
Inline Side-by-side
Showing 8 changed files with 83 additions and 65 deletions
CHANGES.txt
3.1 2023-06-28
--------------
- Tambah bin/payment_report_by_id
3.0.2 2023-05-17 3.0.2 2023-05-17
---------------- ----------------
- PBB kini memahami tabel log_iso dengan struktur universal (JSON). - PBB kini memahami tabel log_iso dengan struktur universal (JSON).
......
payment_report/common.py
...@@ -9,8 +9,6 @@ from datetime import ( ...@@ -9,8 +9,6 @@ from datetime import (
datetime, datetime,
timedelta, timedelta,
) )
from argparse import ArgumentParser
from configparser import ConfigParser
from ISO8583.ISO8583 import BitNotSet from ISO8583.ISO8583 import BitNotSet
from sqlalchemy import ( from sqlalchemy import (
Column, Column,
...@@ -214,6 +212,9 @@ class Conf(BaseConf): ...@@ -214,6 +212,9 @@ class Conf(BaseConf):
pola += '.%f' pola += '.%f'
return datetime.strptime(self.nilai, pola) return datetime.strptime(self.nilai, pola)
def as_int(self):
return int(self.nilai)
def create_session(db_url, debug=False): def create_session(db_url, debug=False):
engine = create_engine(db_url, echo=debug) engine = create_engine(db_url, echo=debug)
...@@ -299,11 +300,11 @@ class BaseApp: ...@@ -299,11 +300,11 @@ class BaseApp:
def get_session_for_save(self): def get_session_for_save(self):
return self.rpt_session return self.rpt_session
def update_from_date(self): def do_sync(self):
q = self.get_payment_query() q = self.get_payment_query()
no = self.offset no = self.offset
found = False found = False
log = getLogger('update_from_date()') log = getLogger('do_sync()')
for pay in q.offset(self.offset).limit(row_limit): for pay in q.offset(self.offset).limit(row_limit):
found = True found = True
no += 1 no += 1
...@@ -351,13 +352,13 @@ class BaseApp: ...@@ -351,13 +352,13 @@ class BaseApp:
self.last.nilai = self.get_last_time() self.last.nilai = self.get_last_time()
with transaction.manager: with transaction.manager:
self.rpt_session.add(self.last) self.rpt_session.add(self.last)
self.rpt_session.flush()
self.rpt_session.expunge_all()
def get_count(self): # Override, please def get_count(self): # Override, please
pass pass
def run(self): def prepare_query_filter(self):
self.last_pay = self.last = None
self.offset = 0
if 'tgl_awal' in self.conf: if 'tgl_awal' in self.conf:
self.tgl_awal = self.conf['tgl_awal'] self.tgl_awal = self.conf['tgl_awal']
else: else:
...@@ -367,12 +368,17 @@ class BaseApp: ...@@ -367,12 +368,17 @@ class BaseApp:
self.tgl_akhir = self.conf['tgl_akhir'] self.tgl_akhir = self.conf['tgl_akhir']
else: else:
self.tgl_akhir = date.today() self.tgl_akhir = date.today()
def run(self):
self.last_pay = self.last = None
self.offset = 0
self.prepare_query_filter()
while True: while True:
self.count = self.get_count() self.count = self.get_count()
if not self.count: if not self.count:
return return
self.start_time = time() self.start_time = time()
found = self.update_from_date() found = self.do_sync()
if not found: if not found:
break break
if self.last_pay and self.last: if self.last_pay and self.last:
...@@ -404,3 +410,15 @@ class BaseApp: ...@@ -404,3 +410,15 @@ class BaseApp:
raise Exception( raise Exception(
f'Invoice ID {invoice_id} va_type {inv.va_type} ' f'Invoice ID {invoice_id} va_type {inv.va_type} '
'belum dipahami, perbaiki script') 'belum dipahami, perbaiki script')
class BaseAppById(BaseApp):
def prepare_query_filter(self): # Override
if self.conf.get('start_id'):
self.last_id = self.conf.get('start_id') - 1
else:
self.last = self.get_last_id(self.conf_name)
self.last_id = self.last.as_int()
def get_last_time(self): # Override
return str(self.last_pay.id)
payment_report/scripts/log2iso_ciamis.py → payment_report/log2iso.py
...@@ -6,35 +6,19 @@ from sqlalchemy import func ...@@ -6,35 +6,19 @@ from sqlalchemy import func
from zope.sqlalchemy import register from zope.sqlalchemy import register
from opensipkd.waktu import create_datetime from opensipkd.waktu import create_datetime
from opensipkd.string import FixLength from opensipkd.string import FixLength
from ..models import Common from .models import Common
from ..log_models import LogFile from .log_models import LogFile
from .common import ( from .common import (
BaseApp, BaseAppById,
my_registry, my_registry,
append_csv,
create_session, create_session,
InvalidSource, InvalidSource,
) )
def get_parser(): class App(BaseAppById):
pars = ArgumentParser() def __init__(self, conf):
pars.add_argument('conf') super().__init__(conf)
pars.add_argument('--update-from-id', type=int)
pars.add_argument('--debug-sql', action='store_true')
return pars
def get_option(argv):
pars = get_parser()
return pars.parse_args(argv)
class App(BaseApp):
def __init__(self, argv):
super().__init__(argv)
if not self.pid:
return
class Log(self.models.Log, Common): class Log(self.models.Log, Common):
pass pass
...@@ -48,9 +32,6 @@ class App(BaseApp): ...@@ -48,9 +32,6 @@ class App(BaseApp):
self.pajak = 'pbb' self.pajak = 'pbb'
register(self.prod_session) register(self.prod_session)
def get_option(self, argv): # Override
return get_option(argv)
def get_session_for_save(self): # Override def get_session_for_save(self): # Override
return self.prod_session return self.prod_session
...@@ -83,7 +64,12 @@ class App(BaseApp): ...@@ -83,7 +64,12 @@ class App(BaseApp):
raise InvalidSource(f'ID {log_file.id} bukan payment') raise InvalidSource(f'ID {log_file.id} bukan payment')
except BitNotSet: except BitNotSet:
raise InvalidSource(f'ID {log_file.id} bit 3 tidak ada') raise InvalidSource(f'ID {log_file.id} bit 3 tidak ada')
waktu = iso.get_transaction_time() # filename = 'sent_220103_log.txt'
tahun = '20' + log_file.filename[5:7] # 2022
tahun = int(tahun)
raw = iso.getBit(13) + iso.getBit(12)
iso.transaction_time.set_raw(raw)
waktu = iso.transaction_time.get_value(tahun)
d = dict( d = dict(
id=log_file.id, mti=iso.getMTI(), created=waktu, id=log_file.id, mti=iso.getMTI(), created=waktu,
bits=iso.get_values()) bits=iso.get_values())
...@@ -102,29 +88,3 @@ class App(BaseApp): ...@@ -102,29 +88,3 @@ class App(BaseApp):
profile.set_raw(iso.getBit(62)) profile.set_raw(iso.getBit(62))
d['bits_data'] = {62: profile.to_dict()} d['bits_data'] = {62: profile.to_dict()}
return d return d
def get_last_time(self): # Override
return str(self.last_pay.id)
def run_payment(self): # Override
self.last_pay = None
self.offset = 0
if self.option.update_from_id is None:
self.last = self.get_last_id(self.conf_name)
self.last_id = self.last.as_int()
else:
self.last = None
self.last_id = self.option.update_from_id - 1
self.count = self.get_count()
self.start_time = time()
while True:
found = self.update_from_date()
if not found:
break
if self.last_pay and self.last:
self.update_last()
def main(argv=sys.argv[1:]):
app = App(argv)
app.run()
payment_report/log_models.py
...@@ -15,3 +15,4 @@ class LogFile(Base): ...@@ -15,3 +15,4 @@ class LogFile(Base):
id = Column(Integer, primary_key=True) id = Column(Integer, primary_key=True)
line_id = Column(String(64), nullable=False, unique=True) line_id = Column(String(64), nullable=False, unique=True)
line = Column(Text, nullable=False) line = Column(Text, nullable=False)
filename = Column(String(32), nullable=False)
payment_report/scripts/log2db_ciamis.py
...@@ -13,7 +13,7 @@ from sqlalchemy.exc import ( ...@@ -13,7 +13,7 @@ from sqlalchemy.exc import (
OperationalError, OperationalError,
) )
from ..log_models import LogFile from ..log_models import LogFile
from .tools import BacaFile from ..tools import BacaFile
RE_CONNECTED = re.compile('connected$') RE_CONNECTED = re.compile('connected$')
...@@ -29,7 +29,7 @@ def read_log(log_file, db_url): ...@@ -29,7 +29,7 @@ def read_log(log_file, db_url):
print([line]) print([line])
line_id = sha256(line.encode('utf-8')).hexdigest() line_id = sha256(line.encode('utf-8')).hexdigest()
while True: while True:
row = LogFile(line_id=line_id, line=line) row = LogFile(line_id=line_id, line=line, filename=short_filename)
try: try:
db_session.add(row) db_session.add(row)
db_session.flush() db_session.flush()
...@@ -44,6 +44,7 @@ def read_log(log_file, db_url): ...@@ -44,6 +44,7 @@ def read_log(log_file, db_url):
print('Tunggu, mungkin sedang maintenance ...') print('Tunggu, mungkin sedang maintenance ...')
sleep(10) sleep(10)
short_filename = os.path.split(log_file)[-1]
engine = create_engine(db_url) engine = create_engine(db_url)
factory = sessionmaker(bind=engine) factory = sessionmaker(bind=engine)
db_session = factory() db_session = factory()
...@@ -72,6 +73,7 @@ def main(argv=sys.argv[1:]): ...@@ -72,6 +73,7 @@ def main(argv=sys.argv[1:]):
dir_name = conf.get('main', 'log_dir') dir_name = conf.get('main', 'log_dir')
pola = os.path.join(dir_name, 'sent_*_log.txt') pola = os.path.join(dir_name, 'sent_*_log.txt')
files = glob(pola) files = glob(pola)
files.sort()
for log_file in files: for log_file in files:
bak_file = log_file[:-3] + 'bak' bak_file = log_file[:-3] + 'bak'
read_log(log_file, db_url) read_log(log_file, db_url)
......
payment_report/scripts/sync.py
...@@ -61,8 +61,8 @@ def get_option(argv): ...@@ -61,8 +61,8 @@ def get_option(argv):
return pars.parse_args(argv) return pars.parse_args(argv)
def main(argv=sys.argv): def main(argv=sys.argv[1:]):
option = get_option(argv[1:]) option = get_option(argv)
conf = ConfigParser() conf = ConfigParser()
conf.read(option.conf) conf.read(option.conf)
cf = dict(conf.items('main')) cf = dict(conf.items('main'))
......
payment_report/scripts/sync_id.py 0 → 100644
import sys
from configparser import ConfigParser
from argparse import ArgumentParser
from opensipkd.iso8583.bjb.scripts.common import get_module_object
from .logger import setup_logging
from .sync import make_pid_file
def get_parser():
pars = ArgumentParser()
pars.add_argument('conf')
pars.add_argument('--update-from-id', type=int)
return pars
def get_option(argv):
pars = get_parser()
return pars.parse_args(argv)
def main(argv=sys.argv[1:]):
option = get_option(argv)
conf = ConfigParser()
conf.read(option.conf)
cf = dict(conf.items('main'))
if not make_pid_file(cf['pid_file']):
return
setup_logging(option.conf)
if option.update_from_id:
cf['start_id'] = option.update_from_id
module = get_module_object(conf.get('main', 'module'))
app = module.App(cf)
app.run()
setup.py
...@@ -36,10 +36,10 @@ setup( ...@@ -36,10 +36,10 @@ setup(
'console_scripts': [ 'console_scripts': [
'payment_report_init_db = payment_report.scripts.init_db:main', 'payment_report_init_db = payment_report.scripts.init_db:main',
'payment_report = payment_report.scripts.sync:main', 'payment_report = payment_report.scripts.sync:main',
'payment_report_by_id = payment_report.scripts.sync_id:main',
'pbb_log = payment_report.scripts.pbb_log:main', 'pbb_log = payment_report.scripts.pbb_log:main',
'log2iso_init_db = payment_report.scripts.log2iso_init:main', 'log2iso_init_db = payment_report.scripts.log2iso_init:main',
'log2iso = payment_report.scripts.log2iso:main', 'log2iso = payment_report.scripts.log2iso:main',
'log2iso_ciamis = payment_report.scripts.log2iso_ciamis:main',
'log2iso_fortuna = payment_report.scripts.log2iso_fortuna:main', 'log2iso_fortuna = payment_report.scripts.log2iso_fortuna:main',
'log2db = payment_report.scripts.log2db:main', 'log2db = payment_report.scripts.log2db:main',
'log2db_ciamis = payment_report.scripts.log2db_ciamis:main', 'log2db_ciamis = payment_report.scripts.log2db_ciamis:main',
......
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!