feat: enhance department and user management with new CSV data and query optimizations

opensipkd/base/models/departmen.py

@@ -79,48 +79,51 @@ class _Departemen(NamaModel):
                     id """
         else:
             sql = """
-               WITH RECURSIVE dep_tree AS (
-  SELECT
-    id,
-    kode,
-    nama::text AS nama,
-    parent_id,
-    status,
-    0 AS lvl,
-    ARRAY[id] AS path
-  FROM departemen
-
-  UNION ALL
-
-  SELECT
-    c.id,
-    c.kode,
-    ct.nama || '/' || c.nama::text AS nama,
-    c.parent_id,
-    c.status,
-    ct.lvl + 1,
-    ct.path || c.id
-  FROM departemen c
-  JOIN dep_tree ct ON c.parent_id = ct.id
-)
-SELECT DISTINCT ON (kode)
-  id,
-  kode,
-  nama as hierarchy,
-  parent_id,
-  status,
-  lvl,
-  path
-FROM dep_tree
+                WITH RECURSIVE dep_tree AS (
+                SELECT
+                    id,
+                    kode,
+                    nama::text AS nama,
+                    parent_id,
+                    status,
+                    0 AS lvl,
+                    ARRAY[id] AS path
+                FROM departemen
+
+                UNION ALL
+
+                SELECT
+                    c.id,
+                    c.kode,
+                    ct.nama || '/' || c.nama::text AS nama,
+                    c.parent_id,
+                    c.status,
+                    ct.lvl + 1,
+                    ct.path || c.id
+                FROM departemen c
+                JOIN dep_tree ct ON c.parent_id = ct.id
+                )
+                SELECT DISTINCT ON (kode)
+                id,
+                kode,
+                nama as hierarchy,
+                parent_id,
+                status,
+                lvl,
+                path
+                FROM dep_tree
                 """
             #.format(str_where=str_where)
 
             if search:
-                sql = f"{sql} WHERE nama ILIKE '%{search}%' "
+                sql = f"{sql} WHERE nama ILIKE :search "
 
             sql = sql+"""ORDER BY kode, path """
+        if search:
+            return cls.db_session.execute(text(sql), {"search": search}).fetchall()
+        else:
+            return cls.db_session.execute(text(sql)).fetchall()
 
-        return cls.db_session.execute(text(sql)).fetchall()
 
 
 class Departemen(_Departemen, Base):

opensipkd/base/scripts/data/kecamatan.csv

+kode,nama,status,dati2_id/res_dati2.kode
+020,Kecamatan Purwaharja,1,32.79
+010,Kecamatan Banjar,1,32.79
+030,Kecamatan Pataruman,1,32.79
+040,Kecamatan Langensari,1,32.79
\ No newline at end of file

opensipkd/base/scripts/data/kelurahan.csv

+kode,nama,status,kecamatan_id/res_kecamatan.kode
+010.001,Kelurahan Situbatu,1,010
+010.002,Desa Neglasari,1,010
+010.003,Desa Cibeureum,1,010
+010.004,Desa Balokang,1,010
+010.005,Kelurahan Banjar,1,010
+010.006,Kelurahan Mekarsari,1,010
+010.007,Desa Jajawar,1,010
+020.001,Kelurahan Purwaharja,1,020
+020.002,Kelurahan Karangpanimbal,1,020
+020.003,Desa Raharja,1,020
+020.004,Desa Mekarharja,1,020
+030.001,Desa Binangun,1,030
+030.002,Desa Batulawang,1,030
+030.003,Desa Karyamukti,1,030
+030.004,Desa Mulyasari,1,030
+030.005,Kelurahan Pataruman,1,030
+030.006,Kelurahan Hegarsari,1,030
+030.007,Desa Sukamukti,1,030
+030.008,Desa Sinartanjung,1,030
+040.001,Desa Kujangsari,1,040
+040.002,Kelurahan Bojongkantong,1,040
+040.003,Desa Rejasari,1,040
+040.004,Desa Langensari,1,040
+040.005,Desa Langensari,1,040
+040.006,Kelurahan Muktisari,1,040
+040.007,Desa Waringisari,1,040
\ No newline at end of file

opensipkd/base/scripts/data/routes.csv

@@ -16,6 +16,7 @@ base-user-add,/user/add,base,user,,,,user-edit,base-user,User Add,1,0,,,form6.pt
 base-user-edit,/user/{id}/edit,base,user,,,,user-edit,base-user,User Edit,1,0,,,form6.pt,
 base-user-view,/user/{id}/view,base,user,,,,user-view,base-user,User View,1,0,,,form6.pt,
 base-user-delete,/user/{id}/delete,base,user,,,,user-edit,base-user,User Delete,1,0,,,form6.pt,
+base-user-upload,/user/upload,base,user,,,,user-edit,base-user,User Upload,1,0,,,form6.pt,
 base-partner,/partner,base,partner,,view_list,,admin,base-admin,Partner,1,0,,1,form.pt,
 base-partner-act,/partner/{act}/act,base,partner,,,,admin,base-partner,Partner Action,1,0,,,json,
 base-partner-add,/partner/add,base,partner,,,,admin,base-partner,Partner Add,1,0,,,form6.pt,
@@ -82,7 +83,7 @@ base-user-area-add,/user/area/add,base,user_area,,,,admin,base-user-area,User-ar
 base-user-area-edit,/user/area/{id}/edit,base,user_area,,,,admin,base-user-area,User-area Edit,1,0,,,form6.pt,
 base-user-area-view,/user/area/{id}/view,base,user_area,,,,admin,base-user-area,User-area View,1,0,,,form6.pt,
 base-user-area-delete,/user/area/{id}/delete,base,user_area,,,,admin,base-user-area,User-area Delete,1,0,,,form6.pt,
-base-test,/test,base,web_test_http,,view_add,,,,Test HTTP,1,0,,,test.pt
+,/test,base,web_test_http,,view_add,,,,Test HTTP,1,0,,,test.pt
 base-partner-departemen,/partner/departemen,base,posisi,,view_list,,admin,base-admin,Partner-departemen,1,0,,1,form.pt,
 base-partner-departemen-act,/partner/departemen/{act}/act,base,posisi,,,,admin,base-partner-departemen,Partner-departemen Action,1,0,,,json,
 base-partner-departemen-add,/partner/departemen/add,base,posisi,,,,admin,base-partner-departemen,Partner-departemen Add,1,0,,,form6.pt,
@@ -115,11 +116,3 @@ base-printer-add,/printer/add,base,printers,,,,printers,base-printer,Printer Add
 base-printer-edit,/printer/{id}/edit,base,printers,,,,printers,base-printer,Printer Edit,1,0,,,form6.pt,
 base-printer-view,/printer/{id}/view,base,printers,,,,printers,base-printer,Printer View,1,0,,,form6.pt,
 base-printer-delete,/printer/{id}/delete,base,printers,,,,printers,base-printer,Printer Delete,1,0,,,form6.pt,
-
-base-xhr-test,/xhr/test,,xhr_test,,view_list,,admin,base-admin,Test,1,0,,1,test_list.pt
-base-xhr-test-act,/xhr/test/{act}/act,base,xhr_test,,,,admin,base-test,Test Action,1,0,,,json,
-base-xhr-test-add,/xhr/test/add,base,xhr_test,,,,admin,base-test,Test Add,1,0,,,test_form.pt,
-base-xhr-test-edit,/xhr/test/{id}/edit,base,xhr_test,,,,admin,base-test,Test Edit,1,0,,,test_form.pt,
-base-xhr-test-view,/xhr/test/{id}/view,base,xhr_test,,,,admin,base-test,Test View,1,0,,,test_form.pt,
-base-xhr-test-delete,/xhr/test/{id}/delete,base,xhr_test,,,,admin,base-test,Test Delete,1,0,,,test_form.pt,
-base-xhr-test-api,/xhr/test/api,,xhr_test,ViewsApi,,,admin,,API Test,1,2,,,
\ No newline at end of file

opensipkd/base/scripts/data/users_2.csv

+email,status,user_name,user_password,registered_date
+test@localhost,1,test,a,
\ No newline at end of file

opensipkd/base/scripts/initializedb.py

@@ -202,7 +202,7 @@ def append_csv(table, filename, keys, get_file_func=get_file,
                db_session=DBSession, base=Base, **args):
     eng = db_session.get_bind()
 
-    update_exist = args.get("update_exist")
+    update_exist = args.get("update_exist") or args.get("update")
     callback = args.get("callback")
     delimiter = args.get("delimiter")
     ext = get_ext(filename).lower()
@@ -347,10 +347,11 @@ def append_csv(table, filename, keys, get_file_func=get_file,
             db_session.flush()
             if user:
                 print("Table: ", table.__name__, filter_)
-                row = db_session.query(User).filter_by(id=row.id).first()
+                user_ = db_session.query(User).filter_by(id=row.id).first()
                 init_model()
-                UserService.set_password(row, password)
-                db_session.add(row)
+                # UserService.set_password(user_, password)
+                user_.password = password
+                db_session.add(user_)
                 db_session.flush()
 
         transaction.commit()  # diperlukan commit per record khususnya untuk yang internal link

opensipkd/base/views/base_views.py

 import logging
-from cgi import FieldStorage
+# from cgi import FieldStorage
 import os
 from datetime import datetime
 from email.utils import parseaddr
+from multipart import MultipartPart
 import lxml
 from webob.multidict import MultiDict
 
@@ -830,7 +831,7 @@ class BaseView(object):
 
     def xls_response(self, **kwargs):
         value = self.xls_data()
-        return xls_response(self.req, value)
+        return xls_response(self.req, value, startrow=4)
 
     def csv_response(self, **kwargs):
         value = self.xls_data()
@@ -1096,7 +1097,7 @@ class BaseView(object):
             cloned = self.req.POST.items()
             control = []
             for ctrl in cloned:
-                if isinstance(ctrl[1], FieldStorage):
+                if isinstance(ctrl[1], MultipartPart): # FieldStorage
                     control.append(
                         ("__start__", f"{ctrl[0]}:mapping"))
                     control.append(("upload", ctrl[1]))
@@ -1149,7 +1150,7 @@ class BaseView(object):
                     cloned = self.req.POST.items()
                     control = []
                     for ctrl in cloned:
-                        if isinstance(ctrl[1], FieldStorage):
+                        if isinstance(ctrl[1], File):  # FieldStorage
                             control.append(
                                 ("__start__", f"{ctrl[0]}:mapping"))
                             control.append(("upload", ctrl[1]))
@@ -1356,7 +1357,8 @@ class BaseView(object):
                     cloned = request.POST.items()
                     controls = []
                     for ctrl in cloned:
-                        if isinstance(ctrl[1], FieldStorage):
+                        # FieldStorageFieldStorage):
+                        if isinstance(ctrl[1], MultipartPart):
                             controls.append(
                                 ("__start__", f"{ctrl[0]}:mapping"))
                             controls.append(("upload", ctrl[1]))

opensipkd/base/views/departemen.py

 import colander
 from deform import (widget, )
+from sqlalchemy.orm import aliased
 from opensipkd.models import DBSession, Departemen
 from opensipkd.tools.buttons import btn_upload
 from ..views import BaseView
@@ -83,6 +84,7 @@ class EditSchema(AddSchema):
     id = colander.SchemaNode(colander.String(), missing=colander.drop,
                              widget=widget.HiddenWidget(readonly=True))
 
+subq = aliased(Departemen)
 
 class ListSchema(colander.Schema):
     id = colander.SchemaNode(colander.String(), title="Action", visible=False)
@@ -94,12 +96,12 @@ class ListSchema(colander.Schema):
                                  widget=widget.CheckboxWidget())
     level_id = colander.SchemaNode(
         colander.Integer(), title="Level", width='40pt')
-    parent_id = colander.SchemaNode(colander.String(), title="Induk")
+    parent_id = colander.SchemaNode(colander.String(), title="Induk", field=subq.nama)
     # company_nm = colander.SchemaNode(colander.String(), title="Company")
-    def after_bind(self, schema, kw):
-        request = kw.get('request')
-        schema["parent_id"].widget = widget.Select2Widget(
-            values=get_departemen_list())
+    # def after_bind(self, schema, kw):
+    #     request = kw.get('request')
+    #     schema["parent_id"].widget = widget.Select2Widget(
+    #         values=get_departemen_list())
 
 class Views(BaseView):
     def __init__(self, request):
@@ -110,7 +112,7 @@ class Views(BaseView):
         self.table = Departemen
         self.list_route = 'base-departemen'
         self.form_scripts = ""
-        self.list_buttons = self.list_buttons + (btn_upload,)
+        self.list_buttons = self.list_buttons + self.list_report+(btn_upload,)
 
     def form_validator(self, form, value):
         def err_kode():
@@ -173,25 +175,44 @@ class Views(BaseView):
             values["parent_id"] = None
         row = super().save_request(values, row)
         return row
-    def view_act(self):
-        request = self.req
-        url_dict = request.matchdict
-        if url_dict['act'] == 'grid':
-            query = Departemen.cte_get()
-            data = [{"id": d.id, "kode": d.kode, 
-                     "nama": d.hierarchy.startswith(
-                         '/') and d.hierarchy[1:] or d.hierarchy,
-                     "status": d.status,
-                     "level_id": d.lvl, "parent_id": d.parent_id} for d in query]
-            return {
-                "draw": "1",
-                "recordsTotal": len(data),
-                "recordsFiltered": len(data),
-                "data": data}
-        else:
-            return self.next_act()
+    
+    def list_join(self, query, **kwargs):
+        return query.outerjoin(subq, subq.id == self.table.parent_id)
+    
+    # def get_list(self, **kwargs):
+    #     query = Departemen.cte_get()
+    #     data = [{"id": d.id, "kode": d.kode,
+    #              "nama": d.nama,
+    #             #  d.hierarchy.startswith(
+    #             #      '/') and d.hierarchy[1:] or d.hierarchy,
+    #              "status": d.status,
+    #              "level_id": d.lvl, 
+    #              "parent_id": d.parent_id} for d in query]
+    #     return {
+    #         "draw": "1",
+    #         "recordsTotal": len(data),
+    #         "recordsFiltered": len(data),
+    #         "data": data}
 
-    def next_act(self):
+    # def view_act(self):
+    #     request = self.req
+    #     url_dict = request.matchdict
+    #     if url_dict['act'] == 'grid':
+    #         query = Departemen.cte_get()
+    #         data = [{"id": d.id, "kode": d.kode, 
+    #                  "nama": d.hierarchy.startswith(
+    #                      '/') and d.hierarchy[1:] or d.hierarchy,
+    #                  "status": d.status,
+    #                  "level_id": d.lvl, "parent_id": d.parent_id} for d in query]
+    #         return {
+    #             "draw": "1",
+    #             "recordsTotal": len(data),
+    #             "recordsFiltered": len(data),
+    #             "data": data}
+    #     else:
+    #         return self.next_act()
+
+    def next_act(self, **kwargs):
         request = self.req
         params = request.params
         url_dict = request.matchdict
@@ -211,9 +232,7 @@ class Views(BaseView):
                 r.append(d)
             return r
 
-    
-
-    def get_values(self, row, values=None):
+    def get_values(self, row, values=None, istime=False, null=False):
         if not values:
             values = row.to_dict()
         if 'parent_id' in values and values['parent_id']:
@@ -221,6 +240,7 @@ class Views(BaseView):
             values["parent_nm"] = parent.nama
             values["parent_kd"] = parent.kode
         return values
+    
     # def save_upload(self, kode, csv_row):
     #     row = Departemen.query_kode(kode).first()
     #     if not row:

opensipkd/base/views/kecamatan.py

@@ -135,14 +135,16 @@ class Views(BaseView):
         return d
     
     def list_join(self, query):
-        return query.join(ResDati2, ResDati2.id == ResKecamatan.dati2_id)
+        return query.outerjoin(ResDati2, ResDati2.id == ResKecamatan.dati2_id)
 
     # @view_config(route_name='kecamatan-upload',
     #              renderer='templates/form.pt', permission='wilayah')
     def view_upload(self):
         return super().view_upload(exts=(".csv", ".tsv"))
 
-
+    def view_upload(self, **kw):
+        kw["update"]=True
+        return self.view_import(**kw)
     #     @view_config(route_name='kecamatan-view',
     #                  renderer='templates/form.pt', permission='wilayah')
     # def view_view(self):  # row = query_id(request).first()

opensipkd/base/views/user.py

@@ -3,12 +3,12 @@ import re
 
 import colander
 from deform import (widget, )
-from opensipkd.base import BASE_CLASS
 from pyramid.i18n import TranslationStringFactory
 from sqlalchemy import (func )
 from ziggurat_foundations.models.services.user import UserService
+from opensipkd.base import BASE_CLASS
 from opensipkd.tools import create_now
-from opensipkd.tools.buttons import btn_delete
+from opensipkd.tools.buttons import btn_delete, btn_upload
 
 from . import BaseView
 from .user_login import (
@@ -54,13 +54,14 @@ class Views(BaseView):
         self.table = User
         self.edit_schema = EditSchema
         self.add_schema = AddSchema
-        self.list_buttons = (btn_delete,)+self.list_buttons + self.list_report
+        self.list_buttons = (btn_delete, btn_upload)+self.list_buttons + self.list_report 
         path = os.path.dirname(__file__)
         path = os.path.dirname(path)
         self.report_file = os.path.join(path, 'reports', 'users.jrxml')
         self.allow_check = True
         self.check_field = "id"
         self.list_view_field = 'email'
+        self.upload_keys = ["email"]
 
 
     def get_bindings(self, row=None):