Skip to content

aa.gusti / opensipkd-base

Go to a project
Commit 67dc38ef by aagusti
Options

perbaikan append csv detect csv atau tsv

1 parent 87ad06b5
Inline Side-by-side
Showing 27 changed files with 82 additions and 17 deletions
.gitignore:Zone.Identifier 0 → 100644
[ZoneTransfer]
ZoneId=3
CHANGES.txt:Zone.Identifier 0 → 100644
[ZoneTransfer]
ZoneId=3
LICENSE:Zone.Identifier 0 → 100644
[ZoneTransfer]
ZoneId=3
MANIFEST.in:Zone.Identifier 0 → 100644
[ZoneTransfer]
ZoneId=3
README.md:Zone.Identifier 0 → 100644
[ZoneTransfer]
ZoneId=3
README.txt:Zone.Identifier 0 → 100644
[ZoneTransfer]
ZoneId=3
aset.ini:Zone.Identifier 0 → 100644
[ZoneTransfer]
ZoneId=3
base/activate 0 → 100755
File mode changed
bphtb.ini:Zone.Identifier 0 → 100644
[ZoneTransfer]
ZoneId=3
install-nginx.rst:Zone.Identifier 0 → 100644
[ZoneTransfer]
ZoneId=3
install.txt:Zone.Identifier 0 → 100644
[ZoneTransfer]
ZoneId=3
log.txt:Zone.Identifier 0 → 100644
[ZoneTransfer]
ZoneId=3
log_file_location:Zone.Identifier 0 → 100644
[ZoneTransfer]
ZoneId=3
nasional.ini:Zone.Identifier 0 → 100644
[ZoneTransfer]
ZoneId=3
opensipkd/base/scripts/initializedb.py
...@@ -11,6 +11,7 @@ from opensipkd.models import ( ...@@ -11,6 +11,7 @@ from opensipkd.models import (
User, Route, Eselon, Jabatan, ResProvinsi, ResDati2, ResKecamatan, ResDesa, User, Route, Eselon, Jabatan, ResProvinsi, ResDati2, ResKecamatan, ResDesa,
Menus, Pangkat) Menus, Pangkat)
from opensipkd.models.handlers import LogDBSession from opensipkd.models.handlers import LogDBSession
from opensipkd.tools import get_ext
from pyramid.paster import (get_appsettings, setup_logging, ) from pyramid.paster import (get_appsettings, setup_logging, )
from sqlalchemy import (engine_from_config, select, Table, inspect) from sqlalchemy import (engine_from_config, select, Table, inspect)
from sqlalchemy import text from sqlalchemy import text
...@@ -138,14 +139,25 @@ def restore_csv(table, filename, get_file_func=get_file, db_session=DBSession): ...@@ -138,14 +139,25 @@ def restore_csv(table, filename, get_file_func=get_file, db_session=DBSession):
# masih memungkinkan update yg sudah ada dgn syarat is value dari keys masih sama # masih memungkinkan update yg sudah ada dgn syarat is value dari keys masih sama
# sperti salah route url asalkan kode msh sama # sperti salah route url asalkan kode msh sama
def append_csv(table, filename, keys, get_file_func=get_file, def append_csv(table, filename, keys, get_file_func=get_file,
db_session=DBSession, update_exist=False, delimiter=",", **args): db_session=DBSession, **args):
insp = inspect(DBSession.connection()) update_exist = args.get("update_exist")
callback = args.get("callback") callback = args.get("callback")
delimiter = args.get("delimiter")
ext = get_ext(filename).lower()
log.debug(f"Extension: {ext.strip()}")
log.debug(f"Extension: {ext.strip()=='.tsv'}")
if not delimiter:
delimiter = ","
if ext.strip() == '.tsv':
delimiter = "\t"
log.debug(f"Delimiter: {delimiter}")
insp = inspect(DBSession.connection())
columns_table = insp.get_columns(table.__tablename__) columns_table = insp.get_columns(table.__tablename__)
fields = {} fields = {}
for c in columns_table: for c in columns_table:
fields[c["name"]] = c["type"] fields[c["name"]] = c["type"]
# raise Exception(columns_table)
with get_file_func(filename) as f: with get_file_func(filename) as f:
reader = csv.DictReader(f, delimiter=delimiter) reader = csv.DictReader(f, delimiter=delimiter)
...@@ -154,7 +166,7 @@ def append_csv(table, filename, keys, get_file_func=get_file, ...@@ -154,7 +166,7 @@ def append_csv(table, filename, keys, get_file_func=get_file,
is_first = True is_first = True
fmap = dict() fmap = dict()
for cf in reader: for cf in reader:
log.info(cf) log.debug(f"Column Field: {cf}")
if is_first: if is_first:
is_first = False is_first = False
for fname in cf.keys(): for fname in cf.keys():
...@@ -185,10 +197,11 @@ def append_csv(table, filename, keys, get_file_func=get_file, ...@@ -185,10 +197,11 @@ def append_csv(table, filename, keys, get_file_func=get_file,
fmap[fname] = fname_orig fmap[fname] = fname_orig
data = dict() data = dict()
for fname in cf: for fname in cf:
if not fname: if not fname:
continue continue
# Buka Tabel Foreign
if fname in foreigns: if fname in foreigns:
foreign_table, foreign_field = foreigns[fname] foreign_table, foreign_field = foreigns[fname]
value = cf[fname] value = cf[fname]
...@@ -207,10 +220,15 @@ def append_csv(table, filename, keys, get_file_func=get_file, ...@@ -207,10 +220,15 @@ def append_csv(table, filename, keys, get_file_func=get_file,
# connection.close() # connection.close()
else: else:
value = cf[fname] value = cf[fname]
fname_orig = fmap[fname] fname_orig = fmap[fname]
data[fname_orig] = value data[fname_orig] = value
for key in keys: for key in keys:
if key not in data or not data[key]:
raise Exception(f"Field '{key}' wajib ada")
filter_[key] = data[key] filter_[key] = data[key]
q = db_session.query(table).filter_by(**filter_) q = db_session.query(table).filter_by(**filter_)
row = q.first() row = q.first()
if row: if row:
...@@ -234,6 +252,12 @@ def append_csv(table, filename, keys, get_file_func=get_file, ...@@ -234,6 +252,12 @@ def append_csv(table, filename, keys, get_file_func=get_file,
val = (val == 'true' or val == '1' or val == 1) and True or False val = (val == 'true' or val == '1' or val == 1) and True or False
setattr(row, fname_orig, val) setattr(row, fname_orig, val)
# Penambahan checking field nullable false wajib ada datanya 2024-09-05
for c in columns_table:
if (not c["nullable"] and (c["name"] not in data or not data[c["name"]])
and c["name"] != "id"):
raise Exception(f"Field '{c['name']}' wajib ada {data}")
db_session.add(row) db_session.add(row)
db_session.flush() db_session.flush()
if user: if user:
......
opensipkd/base/views/base_views.py
...@@ -334,7 +334,8 @@ class BaseView(object): ...@@ -334,7 +334,8 @@ class BaseView(object):
buttons = (btn_post, btn_close) buttons = (btn_post, btn_close)
return self.view_view(buttons=buttons) return self.view_view(buttons=buttons)
def view_upload(self, exts=('.png', '.ico'), delimiter=",", **args): def view_upload(self, exts=('.png', '.ico'), **args):
delimiter = args.get("delimiter")
bindings = self.get_bindings() bindings = self.get_bindings()
form = self.get_form(self.upload_schema, bindings=bindings) form = self.get_form(self.upload_schema, bindings=bindings)
resources = form.get_widget_resources() resources = form.get_widget_resources()
......
opensipkd/base/views/departemen.py
...@@ -18,7 +18,7 @@ SESS_EDIT_FAILED = 'Edit departemen gagal' ...@@ -18,7 +18,7 @@ SESS_EDIT_FAILED = 'Edit departemen gagal'
def get_departemen_list(): def get_departemen_list():
r = [("","--Pilih Departemen--")] r = [("", "--Pilih Departemen--")]
q = DBSession.query(Departemen).order_by(Departemen.nama) q = DBSession.query(Departemen).order_by(Departemen.nama)
for row in q: for row in q:
g = (str(row.id), (f"{row.kode}/ {row.nama}")) g = (str(row.id), (f"{row.kode}/ {row.nama}"))
...@@ -333,10 +333,10 @@ class ViewDepartemen(BaseView): ...@@ -333,10 +333,10 @@ class ViewDepartemen(BaseView):
return super(ViewDepartemen, self).view_delete() return super(ViewDepartemen, self).view_delete()
@view_config(route_name='departemen-upload', @view_config(route_name='departemen-upload',
renderer='templates/departemen/upload.pt', renderer='templates/form.pt',
permission='departemen') permission='departemen')
def view_upload(self): def view_upload(self):
return super().view_upload(exts=('.csv',), delimiter="\t") return super().view_upload(exts=('.csv', '.tsv'), delimiter="\t")
# request = self.req # request = self.req
# form = self.get_form(UploadSchema) # form = self.get_form(UploadSchema)
......
opensipkd/base/views/desa.py
import colander import colander
from deform import (widget, ) from deform import (widget, )
from opensipkd.models import DBSession, ResDesa, kategori_desa, ResKecamatan, ResProvinsi, ResDati2
from opensipkd.tools.buttons import btn_upload, btn_close, btn_add from opensipkd.tools.buttons import btn_upload, btn_close, btn_add
from pyramid.i18n import TranslationStringFactory from pyramid.i18n import TranslationStringFactory
from pyramid.view import (view_config, ) from pyramid.view import (view_config, )
...@@ -7,8 +8,8 @@ from pyramid.view import (view_config, ) ...@@ -7,8 +8,8 @@ from pyramid.view import (view_config, )
from .dati2 import dati2_widget from .dati2 import dati2_widget
from .kecamatan import kecamatan_widget from .kecamatan import kecamatan_widget
from .provinsi import provinsi_widget from .provinsi import provinsi_widget
from opensipkd.models import DBSession, ResDesa, kategori_desa, ResKecamatan, ResProvinsi, ResDati2
from ..views import BaseView from ..views import BaseView
_ = TranslationStringFactory("opensipkd") _ = TranslationStringFactory("opensipkd")
SESS_ADD_FAILED = 'Tambah desa gagal' SESS_ADD_FAILED = 'Tambah desa gagal'
...@@ -179,4 +180,4 @@ class ViewDesa(BaseView): ...@@ -179,4 +180,4 @@ class ViewDesa(BaseView):
@view_config(route_name='desa-upload', @view_config(route_name='desa-upload',
renderer='templates/form.pt', permission='wilayah') renderer='templates/form.pt', permission='wilayah')
def view_upload(self): def view_upload(self):
return super(ViewDesa, self).view_upload(exts=('.csv',)) return super(ViewDesa, self).view_upload(exts=('.csv', ".tsv"))
opensipkd/base/views/kecamatan.py
import colander import colander
from deform import (widget, ) from deform import (widget, )
from opensipkd.tools.buttons import btn_upload, btn_close, btn_add
from pyramid.i18n import TranslationStringFactory
from opensipkd.base.views.provinsi import provinsi_widget from opensipkd.base.views.provinsi import provinsi_widget
from opensipkd.models import DBSession, ResKecamatan, ResDati2, ResProvinsi from opensipkd.models import DBSession, ResKecamatan, ResDati2, ResProvinsi
from opensipkd.tools.buttons import btn_upload, btn_close, btn_add
from pyramid.i18n import TranslationStringFactory
from pyramid.view import (view_config, ) from pyramid.view import (view_config, )
from . import widget_os from . import widget_os
...@@ -172,4 +171,4 @@ class Views(BaseView): ...@@ -172,4 +171,4 @@ class Views(BaseView):
@view_config(route_name='kecamatan-upload', @view_config(route_name='kecamatan-upload',
renderer='templates/form.pt', permission='wilayah') renderer='templates/form.pt', permission='wilayah')
def view_upload(self): def view_upload(self):
return super(Views, self).view_upload(exts=(".csv",)) return super(Views, self).view_upload(exts=(".csv", ".tsv"))
opensipkd/models/base.py
...@@ -175,7 +175,7 @@ class StandarModel(DefaultModel): ...@@ -175,7 +175,7 @@ class StandarModel(DefaultModel):
class KodeModel(StandarModel): class KodeModel(StandarModel):
kode = Column(String(32)) kode = Column(String(32), nullable=False)
@classmethod @classmethod
def query_kode(cls, kode, db_session=DBSession): def query_kode(cls, kode, db_session=DBSession):
...@@ -203,7 +203,7 @@ class UraianModel(StandarModel): ...@@ -203,7 +203,7 @@ class UraianModel(StandarModel):
class NamaModel(KodeModel): class NamaModel(KodeModel):
nama = Column(String(128)) nama = Column(String(128), nullable=False)
@classmethod @classmethod
def query_nama(cls, nama, db_session=DBSession): def query_nama(cls, nama, db_session=DBSession):
......
requirements.txt:Zone.Identifier 0 → 100644
[ZoneTransfer]
ZoneId=3
requires.txt:Zone.Identifier 0 → 100644
[ZoneTransfer]
ZoneId=3
setup.cfg:Zone.Identifier 0 → 100644
[ZoneTransfer]
ZoneId=3
setup.py:Zone.Identifier 0 → 100644
[ZoneTransfer]
ZoneId=3
test.ini:Zone.Identifier 0 → 100644
[ZoneTransfer]
ZoneId=3
test2.ini:Zone.Identifier 0 → 100644
[ZoneTransfer]
ZoneId=3
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!