perbaikan append csv detect csv atau tsv

.gitignore:Zone.Identifier

+[ZoneTransfer]
+ZoneId=3

CHANGES.txt:Zone.Identifier

+[ZoneTransfer]
+ZoneId=3

LICENSE:Zone.Identifier

+[ZoneTransfer]
+ZoneId=3

MANIFEST.in:Zone.Identifier

+[ZoneTransfer]
+ZoneId=3

README.md:Zone.Identifier

+[ZoneTransfer]
+ZoneId=3

README.txt:Zone.Identifier

+[ZoneTransfer]
+ZoneId=3

aset.ini:Zone.Identifier

+[ZoneTransfer]
+ZoneId=3

bphtb.ini:Zone.Identifier

+[ZoneTransfer]
+ZoneId=3

install-nginx.rst:Zone.Identifier

+[ZoneTransfer]
+ZoneId=3

install.txt:Zone.Identifier

+[ZoneTransfer]
+ZoneId=3

log.txt:Zone.Identifier

+[ZoneTransfer]
+ZoneId=3

log_file_location:Zone.Identifier

+[ZoneTransfer]
+ZoneId=3

nasional.ini:Zone.Identifier

+[ZoneTransfer]
+ZoneId=3

opensipkd/base/scripts/initializedb.py

@@ -11,6 +11,7 @@ from opensipkd.models import (
     User, Route, Eselon, Jabatan, ResProvinsi, ResDati2, ResKecamatan, ResDesa,
     Menus, Pangkat)
 from opensipkd.models.handlers import LogDBSession
+from opensipkd.tools import get_ext
 from pyramid.paster import (get_appsettings, setup_logging, )
 from sqlalchemy import (engine_from_config, select, Table, inspect)
 from sqlalchemy import text
@@ -138,14 +139,25 @@ def restore_csv(table, filename, get_file_func=get_file, db_session=DBSession):
 # masih memungkinkan update yg sudah ada dgn syarat is value dari keys masih sama
 # sperti salah route url asalkan kode msh sama
 def append_csv(table, filename, keys, get_file_func=get_file,
-               db_session=DBSession, update_exist=False, delimiter=",", **args):
-    insp = inspect(DBSession.connection())
+               db_session=DBSession, **args):
+    update_exist = args.get("update_exist")
     callback = args.get("callback")
+    delimiter = args.get("delimiter")
+    ext = get_ext(filename).lower()
+    log.debug(f"Extension: {ext.strip()}")
+    log.debug(f"Extension: {ext.strip()=='.tsv'}")
+    if not delimiter:
+        delimiter = ","
+        if ext.strip() == '.tsv':
+            delimiter = "\t"
+
+    log.debug(f"Delimiter: {delimiter}")
+    insp = inspect(DBSession.connection())
     columns_table = insp.get_columns(table.__tablename__)
-
     fields = {}
     for c in columns_table:
         fields[c["name"]] = c["type"]
+    # raise Exception(columns_table)
 
     with get_file_func(filename) as f:
         reader = csv.DictReader(f, delimiter=delimiter)
@@ -154,7 +166,7 @@ def append_csv(table, filename, keys, get_file_func=get_file,
         is_first = True
         fmap = dict()
         for cf in reader:
-            log.info(cf)
+            log.debug(f"Column Field: {cf}")
             if is_first:
                 is_first = False
                 for fname in cf.keys():
@@ -185,10 +197,11 @@ def append_csv(table, filename, keys, get_file_func=get_file,
 
                     fmap[fname] = fname_orig
             data = dict()
+
             for fname in cf:
                 if not fname:
                     continue
-
+                # Buka Tabel Foreign
                 if fname in foreigns:
                     foreign_table, foreign_field = foreigns[fname]
                     value = cf[fname]
@@ -207,10 +220,15 @@ def append_csv(table, filename, keys, get_file_func=get_file,
                     # connection.close()
                 else:
                     value = cf[fname]
+
                 fname_orig = fmap[fname]
                 data[fname_orig] = value
+
             for key in keys:
+                if key not in data or not data[key]:
+                    raise Exception(f"Field '{key}' wajib ada")
                 filter_[key] = data[key]
+
             q = db_session.query(table).filter_by(**filter_)
             row = q.first()
             if row:
@@ -234,6 +252,12 @@ def append_csv(table, filename, keys, get_file_func=get_file,
                         val = (val == 'true' or val == '1' or val == 1) and True or False
                     setattr(row, fname_orig, val)
 
+            # Penambahan checking field nullable false wajib ada datanya 2024-09-05
+            for c in columns_table:
+                if (not c["nullable"] and (c["name"] not in data or not data[c["name"]])
+                        and c["name"] != "id"):
+                    raise Exception(f"Field '{c['name']}' wajib ada {data}")
+
             db_session.add(row)
             db_session.flush()
             if user:

opensipkd/base/views/base_views.py

@@ -334,7 +334,8 @@ class BaseView(object):
             buttons = (btn_post, btn_close)
         return self.view_view(buttons=buttons)
 
-    def view_upload(self, exts=('.png', '.ico'), delimiter=",", **args):
+    def view_upload(self, exts=('.png', '.ico'), **args):
+        delimiter = args.get("delimiter")
         bindings = self.get_bindings()
         form = self.get_form(self.upload_schema, bindings=bindings)
         resources = form.get_widget_resources()

opensipkd/base/views/departemen.py

@@ -18,7 +18,7 @@ SESS_EDIT_FAILED = 'Edit departemen gagal'
 
 
 def get_departemen_list():
-    r = [("","--Pilih Departemen--")]
+    r = [("", "--Pilih Departemen--")]
     q = DBSession.query(Departemen).order_by(Departemen.nama)
     for row in q:
         g = (str(row.id), (f"{row.kode}/ {row.nama}"))
@@ -333,10 +333,10 @@ class ViewDepartemen(BaseView):
         return super(ViewDepartemen, self).view_delete()
 
     @view_config(route_name='departemen-upload',
-                 renderer='templates/departemen/upload.pt',
+                 renderer='templates/form.pt',
                  permission='departemen')
     def view_upload(self):
-        return super().view_upload(exts=('.csv',), delimiter="\t")
+        return super().view_upload(exts=('.csv', '.tsv'), delimiter="\t")
 
         # request = self.req
         # form = self.get_form(UploadSchema)

opensipkd/base/views/desa.py

 import colander
 from deform import (widget, )
+from opensipkd.models import DBSession, ResDesa, kategori_desa, ResKecamatan, ResProvinsi, ResDati2
 from opensipkd.tools.buttons import btn_upload, btn_close, btn_add
 from pyramid.i18n import TranslationStringFactory
 from pyramid.view import (view_config, )
@@ -7,8 +8,8 @@ from pyramid.view import (view_config, )
 from .dati2 import dati2_widget
 from .kecamatan import kecamatan_widget
 from .provinsi import provinsi_widget
-from opensipkd.models import DBSession, ResDesa, kategori_desa, ResKecamatan, ResProvinsi, ResDati2
 from ..views import BaseView
+
 _ = TranslationStringFactory("opensipkd")
 
 SESS_ADD_FAILED = 'Tambah desa gagal'
@@ -179,4 +180,4 @@ class ViewDesa(BaseView):
     @view_config(route_name='desa-upload',
                  renderer='templates/form.pt', permission='wilayah')
     def view_upload(self):
-        return super(ViewDesa, self).view_upload(exts=('.csv',))
+        return super(ViewDesa, self).view_upload(exts=('.csv', ".tsv"))

opensipkd/base/views/kecamatan.py

 import colander
 from deform import (widget, )
-from opensipkd.tools.buttons import btn_upload, btn_close, btn_add
-from pyramid.i18n import TranslationStringFactory
-
 from opensipkd.base.views.provinsi import provinsi_widget
 from opensipkd.models import DBSession, ResKecamatan, ResDati2, ResProvinsi
+from opensipkd.tools.buttons import btn_upload, btn_close, btn_add
+from pyramid.i18n import TranslationStringFactory
 from pyramid.view import (view_config, )
 
 from . import widget_os
@@ -172,4 +171,4 @@ class Views(BaseView):
     @view_config(route_name='kecamatan-upload',
                  renderer='templates/form.pt', permission='wilayah')
     def view_upload(self):
-        return super(Views, self).view_upload(exts=(".csv",))
+        return super(Views, self).view_upload(exts=(".csv", ".tsv"))

opensipkd/models/base.py

@@ -175,7 +175,7 @@ class StandarModel(DefaultModel):
 
 
 class KodeModel(StandarModel):
-    kode = Column(String(32))
+    kode = Column(String(32), nullable=False)
 
     @classmethod
     def query_kode(cls, kode, db_session=DBSession):
@@ -203,7 +203,7 @@ class UraianModel(StandarModel):
 
 
 class NamaModel(KodeModel):
-    nama = Column(String(128))
+    nama = Column(String(128), nullable=False)
 
     @classmethod
     def query_nama(cls, nama, db_session=DBSession):

python-alternative.rst:Zone.Identifier

+[ZoneTransfer]
+ZoneId=3

requirements.txt:Zone.Identifier

+[ZoneTransfer]
+ZoneId=3

requires.txt:Zone.Identifier

+[ZoneTransfer]
+ZoneId=3

setup.cfg:Zone.Identifier

+[ZoneTransfer]
+ZoneId=3

setup.py:Zone.Identifier

+[ZoneTransfer]
+ZoneId=3

test.ini:Zone.Identifier

+[ZoneTransfer]
+ZoneId=3

test2.ini:Zone.Identifier

+[ZoneTransfer]
+ZoneId=3