Tidak perlu lagi paket Debian imgw

.gitignore

 dist
 *egg-info
+__pycache__
+test-*

MANIFEST.in

 include *.txt *.rst *.py
-recursive-include models *.py
+recursive-include imgw *.csv

README.rst

@@ -2,73 +2,34 @@ Struktur Tabel Instant Messaging Gateway
 ========================================
 
 Repository ini berisi struktur tabel dan beberapa fungsi terkait paket Debian
-bernama ``im-gw``.
+bernama ``im-gw``. Karena paket itu masih Python 2 maka di sini hadir
+penggantinya.
 
-Sebelum Anda menggunakan ini pastikan Anda sudah memasangnya. Berikut
-penjelasan singkatnya.
+Sesuaikan ``agent_job.ini``, lalu buat tabelnya::
 
-Tambahkan GPG key ke *apt sources keyring*::
+  $ ~/env/bin/imgw_init_db agent_job.ini
 
-  $ wget -qO - https://repo.opensipkd.com/debian/gpg.key | sudo apt-key add -
+Login ke database menggunakan ``psql``, lalu::
 
-Buat file ``/etc/apt/sources.list.d/opensipkd.list``::
+  INSERT INTO im.antrian (penerima, pesan)
+  VALUES ('sugiana@gmail.com', 'Uji coba imgw');
 
-  deb [ arch=amd64 ] https://repo.opensipkd.com/debian jessie main
+Pindahkan *job* itu ke file::
 
-Perbarui daftar paket::
+  $ ~/env/bin/imgw_agent_job agent_job.ini
 
-  $ sudo apt-get update
+Hasilnya::
 
-Pasang paketnya::
+  2023-07-03 03:59:40,754 INFO Write /home/sugiana/tmp/mail/job/18.json {'id': 18, 'penerima': 'sugiana@gmail.com', 'pesan': 'Uji coba imgw', 'subject': 'Uji coba imgw', 'name': 'sugiana'}
 
-  $ sudo apt-get install im-gw
+Kirim email berdasarkan file JSON tersebut::
 
-Sesuaikan database profile di ``/etc/im/gw/config.py``, lalu::
+  $ ~/env/bin/imgw_mailer test-mailer.ini
 
-  $ sudo dpkg-reconfigure im-gw
+Hasilnya::
 
-Pasang paket pendukung terkait jalur SMS dan email::
+    2023-07-03 04:04:03,978 DEBUG Login sebagai no-reply@opensipkd.com
+    2023-07-03 04:04:04,051 INFO KIRIM {'id': 18, 'penerima': 'sugiana@gmail.com', 'pesan': 'Uji coba imgw', 'subject': 'Uji coba imgw', 'name': 'sugiana'}
+    2023-07-03 04:04:04,119 INFO {'status': 0, 'jawaban': 'OK'}
 
-  $ sudo apt-get install im-modem im-mail
-
-Setelah itu lanjutkan menggunakan repository ini untuk mengirim email::
-
-  import base64
-  import sys
-  from sqlalchemy import create_engine
-  from sqlalchemy.orm import sessionmaker
-  from imgw.mail import kirim
-
-  db_url = 'postgresql://sugiana:FIXME@localhost/imgw'
-  engine = create_engine(db_url)
-  session_factory = sessionmaker(bind=engine)
-  DBSession = session_factory()
-
-  files = []
-  filename = 'test.pdf'
-  with open(filename, 'rb') as f:
-      content = f.read()
-  if sys.version_info.major == 2:
-      content = base64.encodestring(content)
-  else:
-      content = base64.encodebytes(content)
-  files.append([filename, content])
-
-  p = dict(
-        penerima='sugiana@opensipkd.com', subject='Uji coba',
-        pesan='Hello world', files=files)
-  a = kirim(DBSession, p)
-  DBSession.flush()
-  DBSession.commit()
-
-Sedangkan untuk mengirim SMS::
-
-  from imgw.sms import kirim
-
-  p = dict(penerima='+628123456789', pesan='Hello world')
-  a = kirim(p)
-  DBSession.add(a)
-  DBSession.flush()
-  DBSession.commit()
-
-Selamat mencoba.
+Agar otomatis maka letakkan keduanya di ``cron``.

agent_job.ini

+[main]
+db_url = postgresql://sugiana:FIXME@localhost/imgw
+pid_file = /home/sugiana/tmp/imgw_agent_job.pid
+
+# Direktori pesan yang akan dikirim
+job_dir = /home/sugiana/tmp/mail/job
+
+# Direktori hasil pengiriman
+result_dir = /home/sugiana/tmp/mail/result
+
+
+[formatter_generic]
+format = %(asctime)s %(levelname)s %(message)s
+
+[handler_console]
+class = StreamHandler
+stream = sys.stdout
+formatter = generic
+
+[handler_file]
+class = FileHandler
+filename = /home/sugiana/log/imgw_agent_job.log
+formatter = generic
+
+[logger_root]
+handlers = console, file
+level = INFO

imgw/scripts/agent_job.py

+import sys
+import os
+import json
+from logging import getLogger
+from configparser import ConfigParser
+from textwrap import wrap
+from sqlalchemy import engine_from_config
+from sqlalchemy.schema import CreateSchema
+from sqlalchemy.orm import sessionmaker
+from zope.sqlalchemy import register
+import transaction
+from ..models import (
+    Agent,
+    Antrian,
+    Selesai,
+    Mail,
+    File,
+    )
+from .tools import (
+    make_pid_file,
+    clean_log,
+    )
+from .logger import setup_logging
+
+
+my_registry = dict()
+
+
+def write_job_file(s, mail=None):
+    conf = my_registry['conf']
+    db_session = my_registry['db_session']
+    log = getLogger()
+    job_file = f'{s.id}.json'
+    job_file = os.path.join(conf['job_dir'], job_file)
+    d = dict(
+            id=s.id, penerima=s.penerima, pesan=s.pesan)
+    if mail:
+        d.update(dict(subject=mail.subject, name=mail.name))
+        q = db_session.query(File).filter_by(id=s.id)
+        files = []
+        for f in q.order_by(File.urutan):
+            files.append([f.filename, f.content])
+        if files:
+            d['files'] = files
+    log.info(f'Write {job_file} {clean_log(d)}')
+    json_d = json.dumps(d)
+    with open(job_file, 'w') as f:
+        f.write(json_d)
+    return job_file
+
+
+def create_mail_row(a):
+    name = a.penerima.split('@')[0]
+    # Subject ambil dari penggalan pesan
+    t = wrap(a.pesan, 50)
+    subject = t and t[0] or ''
+    return Mail(id=a.id, name=name, subject=subject)
+
+
+def main(argv=sys.argv[1:]):
+    conf_file = argv[0]
+    conf = ConfigParser()
+    conf.read(conf_file)
+    my_registry['conf'] = cf = dict(conf.items('main'))
+    if not make_pid_file(cf['pid_file']):
+        sys.exit(1)
+    setup_logging(conf_file)
+    engine = engine_from_config(cf, 'db_')
+    factory = sessionmaker(bind=engine)
+    my_registry['db_session'] = db_session = factory()
+    register(db_session)
+    q = db_session.query(Antrian).filter_by(kirim=True)
+    q_mail = db_session.query(Mail)
+    while True:
+        a = q.order_by(Antrian.id).first()
+        if not a:
+            return
+        new_mail = False
+        if a.jalur == 6:  # Mail
+            mail = q_mail.filter_by(id=a.id).first()
+            if not mail:
+                mail = create_mail_row(a)
+                new_mail = True
+        d = a.to_dict()
+        s = Selesai(**d)
+        q_agent = db_session.query(Agent).filter_by(jalur=a.jalur)
+        if s.pengirim:
+            q_agent = q_agent.filter_by(id=s.pengirim)
+        else:
+            q_agent = q_agent.filter_by(status=0)
+        agent = q_agent.first()
+        if agent:
+            s.pengirim = agent.id
+            if agent.status != 0:
+                s.status = -2  # Agent belum hidup
+        else:
+            s.status = -1  # Tidak dapat agent aktif
+        with transaction.manager:
+            q.filter_by(id=a.id).delete()
+            db_session.add(s)
+            if new_mail:
+                db_session.add(mail)
+            db_session.flush()
+            db_session.expunge_all()
+        json_file = write_job_file(s, mail)
+    os.remove(cf['pid_file'])

imgw/scripts/common.py

+import os
+
+
+def is_live(pid):
+    try:
+        os.kill(pid, 0)
+    except OSError:
+        return
+    return True
+
+
+def read_pid_file(filename):
+    try:
+        f = open(filename)
+        s = f.read()
+        f.close()
+        s = s.split()
+        s = s[0]
+        return int(s)
+    except IOError:
+        return
+    except ValueError:
+        return
+    except IndexError:
+        return
+
+
+def write_pid_file(filename):
+    pid = os.getpid()
+    f = open(filename, 'w')
+    f.write(str(pid))
+    f.close()
+    return pid
+
+
+def make_pid_file(filename):
+    pid = read_pid_file(filename)
+    if pid and is_live(pid):
+        print(f'PID saya {pid} masih aktif.')
+        return
+    return write_pid_file(filename)

imgw/scripts/data/jalur.csv

+id,nama
+6,mail

imgw/scripts/data/status.csv

+id,ket
+1,Sedang diproses
+2,Sedang diproses agent
+3,Pesan offline
+4,Pesan dikirim ulang
+0,OK
+-1,Agent tidak terdaftar
+-2,Agent belum hidup
+-3,Agent gagal memproses
+-4,Gangguan network pada agent
+-5,Pesan masih sama
+-6,Pesan tidak dapat diterjemahkan
+-9,Timeout

imgw/scripts/init_db.py

+import sys
+import os
+import csv
+from configparser import ConfigParser
+from sqlalchemy import create_engine
+from sqlalchemy.schema import CreateSchema
+from sqlalchemy.orm import sessionmaker
+from zope.sqlalchemy import register
+import transaction
+from ..models import (
+    Base,
+    Jalur,
+    Status,
+    )
+
+
+my_registry = dict()
+
+
+def get_file(filename):
+    base_dir = os.path.split(__file__)[0]
+    fullpath = os.path.join(base_dir, 'data', filename)
+    return open(fullpath)
+
+
+def append_csv(table, filename, keys):
+    db_session = my_registry['db_session']
+    with get_file(filename) as f:
+        reader = csv.DictReader(f)
+        filter_ = dict()
+        for cf in reader:
+            for key in keys:
+                filter_[key] = cf[key]
+            q = db_session.query(table).filter_by(**filter_)
+            found = q.first()
+            if found:
+                continue
+            row = table()
+            for fieldname in cf:
+                val = cf[fieldname]
+                if not val:
+                    continue
+                setattr(row, fieldname, val)
+            db_session.add(row)
+
+
+def main(argv=sys.argv[1:]):
+    conf_file = argv[0]
+    conf = ConfigParser()
+    conf.read(conf_file)
+    db_url = conf.get('main', 'db_url')
+    engine = create_engine(db_url, echo=True)
+    schema_name = 'im'
+    with engine.connect() as conn:
+        if not conn.dialect.has_schema(conn, schema_name):
+            sql = CreateSchema(schema_name)
+            conn.execute(sql)
+            conn.commit()
+    Base.metadata.create_all(engine)
+    factory = sessionmaker(bind=engine)
+    my_registry['db_session'] = db_session = factory()
+    register(db_session)
+    with transaction.manager:
+        append_csv(Jalur, 'jalur.csv', ['id'])
+        append_csv(Status, 'status.csv', ['id'])

imgw/scripts/logger.py

+from configparser import RawConfigParser
+import logging
+import logging.config
+
+
+def setup_logging(conf_file):
+    conf = RawConfigParser()
+    conf.read(conf_file)
+    d = {
+        'version': 1,
+        'formatters': {},
+        'handlers': {},
+        'loggers': {},
+        }
+    for section in conf.sections():
+        if section.find('formatter_') == 0:
+            name = section.split('formatter_')[1]
+            data = {'format': conf.get(section, 'format')}
+            d['formatters'][name] = data
+        elif section.find('handler_') == 0:
+            name = section.split('handler_')[1]
+            data = {'formatter': conf.get(section, 'formatter')}
+            data['class'] = 'logging.' + conf.get(section, 'class')
+            if conf.has_option(section, 'stream'):
+                data['stream'] = 'ext://' + conf.get(section, 'stream')
+            else:
+                data['filename'] = conf.get(section, 'filename')
+            d['handlers'][name] = data
+        elif section.find('logger_') == 0:
+            name = section.split('logger_')[1]
+            if name == 'root':
+                name = ''
+            data = {'level': conf.get(section, 'level')}
+            value = conf.get(section, 'handlers')
+            data['handlers'] = [x.strip() for x in value.split(',')]
+            d['loggers'][name] = data
+    logging.config.dictConfig(d)

imgw/scripts/mailer.py

+import os
+import sys
+import json
+from logging import getLogger
+from argparse import ArgumentParser
+from configparser import ConfigParser
+import smtplib
+from base64 import b64decode
+from opensipkd.string import exception_message
+from .tools import (
+    make_pid_file,
+    clean_log,
+    create_doc,
+    )
+from .logger import setup_logging
+
+
+registry = dict()
+
+
+def send(doc=None):
+    conf = registry['conf']
+    log = getLogger('send')
+    try:
+        with smtplib.SMTP(conf['host'], conf['port']) as server:
+            if conf['tls']:
+                server.starttls()
+            err = None
+            log.debug(f'Login sebagai {conf["username"]}')
+            try:
+                server.login(conf['username'], conf['password'])
+            except OSError as err:
+                return dict(status=-1, jawaban=str(err).strip())
+            except smtplib.SMTPAuthenticationError as err:
+                return dict(status=-2, jawaban=str(err).strip())
+            if doc:
+                log.info(f'KIRIM {clean_log(doc._data)}')
+                server.sendmail(
+                    conf['username'], doc._data['penerima'], doc.as_string())
+    except ConnectionRefusedError as err:
+        return dict(status=-1, jawaban=str(err).strip())
+    return dict(status=0, jawaban='OK')
+
+
+def job_file_to_doc(filename):
+    log = getLogger('job_file_to_doc')
+    with open(filename) as f:
+        s = f.read()
+    os.remove(filename)
+    try:
+        d = json.loads(s)
+    except Exception:
+        log.error(f'eval({s}): {exception_message()}')
+        return
+    cf = registry['conf']
+    files = []
+    for fname, content in d.get('files', []):
+        data = base64.b64decode(content)
+        files.append([fname, data])
+    doc = create_doc(
+            cf['username'], d['penerima'], d.get('subject'), d['pesan'],
+            cf['name'], d.get('name'), files)
+    # Hack
+    doc._data = d
+    return doc
+
+
+def save_result(d, filename):
+    result_dir = registry['conf']['result_dir']
+    fullpath = os.path.join(result_dir, filename)
+    with open(fullpath, 'w') as f:
+        f.write(json.dumps(d))
+
+
+def do_job():
+    job_dir = registry['conf']['job_dir']
+    job_files = os.listdir(job_dir)
+    if not job_files:
+        return
+    job_files.sort()
+    job_file = os.path.join(job_dir, job_files[0])
+    doc = job_file_to_doc(job_file)
+    if not doc:
+        return
+    log = getLogger('do_job')
+    result_file = os.path.split(job_file)[-1]
+    result = send(doc)
+    if result['status'] == 0:
+        log.info(result)
+    else:
+        log.error(result)
+    save_result(result, result_file)
+
+
+def get_option(argv):
+    pars = ArgumentParser()
+    pars.add_argument('conf')
+    return pars.parse_args(argv)
+
+
+def main(argv=sys.argv[1:]):
+    option = get_option(argv)
+    conf = ConfigParser()
+    conf.read(option.conf)
+    registry['conf'] = cf = dict(conf.items('main'))
+    if not make_pid_file(cf['pid_file']):
+        return
+    setup_logging(option.conf)
+    log = getLogger()
+    cf['tls'] = cf['tls'] == 'true'
+    cf['port'] = int(cf['port'])
+    # result = send()
+    # save_result(result, 'status.json')
+    try:
+        do_job()
+    except Exception as err:
+        d = dict(status=-9, jawaban=exception_message())
+        save_result(d, 'status.json')
+        log.error(d['jawaban'])
+    os.remove(cf['pid_file'])

imgw/scripts/tools.py

+import os
+from email.utils import formatdate
+from email.mime.text import MIMEText
+from email.mime.base import MIMEBase
+from email.mime.multipart import MIMEMultipart
+import mimetypes
+
+
+##########
+# Daemon #
+##########
+def is_live(pid):
+    try:
+        os.kill(pid, 0)
+    except OSError:
+        return
+    return True
+
+
+def read_pid_file(filename):
+    try:
+        with open(filename) as f:
+            s = f.read()
+        s = s.split()
+        s = s[0]
+        return int(s)
+    except IOError:
+        return
+    except ValueError:
+        return
+    except IndexError:
+        return
+
+
+def write_pid_file(filename):
+    pid = os.getpid()
+    with open(filename, 'w') as f:
+        f.write(str(pid))
+    return pid
+
+
+def make_pid_file(filename):
+    pid = read_pid_file(filename)
+    if pid and is_live(pid):
+        print(f'PID saya {pid} masih aktif.')
+        return
+    return write_pid_file(filename)
+
+
+########
+# Mail #
+########
+def nice_name(email, name):
+    if name:
+        return f'{name} <{email}>'
+    return email
+
+
+def file_type(filename):
+    ctype, encoding = mimetypes.guess_type(filename)
+    if ctype is None or encoding is not None:
+        ctype = 'application/octet-stream'
+    maintype, subtype = ctype.split('/', 1)
+    return maintype, subtype
+
+
+def create_doc(
+        from_email, to_email, subject, body, from_name=None, to_name=None,
+        files=[]):
+    msg = doc = MIMEText(body)
+    if files:
+        doc = MIMEMultipart()
+        doc.attach(msg)
+    if subject:
+        doc['Subject'] = subject
+    doc['Date'] = formatdate(localtime=True)
+    if from_email:
+        doc['From'] = nice_name(from_email, from_name)
+    doc['To'] = nice_name(to_email, to_name)
+    for fname, data in files:
+        maintype, subtype = file_type(fname)
+        payload = MIMEBase(maintype, subtype)
+        payload.set_payload(data)
+        encoders.encode_base64(payload)
+        payload.add_header(
+            'Content-Disposition', 'attachment', filename=fname)
+        doc.attach(payload)
+    return doc
+
+
+# Agar log tidak memuat isi attachment
+def clean_log(p):
+    r = {}
+    keys = ['id', 'pengirim', 'from_name', 'penerima', 'pesan', 'jawaban',
+            'tgl_operator', 'subject', 'name', 'To', 'From', 'Subject', 'Body']
+    for key in keys:
+        if key in p:
+            r[key] = p[key]
+    if 'files' not in p:
+        return r
+    files = []
+    for fname, content in p['files']:
+        files.append(fname)
+    r['files'] = files
+    return r

imgw/sms.py

-from .models import Antrian
-
-
-def kirim(p):
-    a = Antrian(kirim=True, jalur=1, penerima=p['penerima'], pesan=p['pesan'])
-    if 'pengirim' in p:
-        a.pengirim = p['pengirim']
-    return a

mailer.ini

+[main]
+host = localhost
+#port = 465 
+port = 587
+tls = true
+username = no-reply@example.com
+password = FIXME 
+name = Robot Contoh
+pid_file = /home/sugiana/tmp/mailer.pid
+
+# Direktori pesan yang akan dikirim
+job_dir = /home/sugiana/tmp/mail/job
+
+# Direktori hasil pengiriman
+result_dir = /home/sugiana/tmp/mail/result
+
+
+[formatter_generic]
+format = %(asctime)s %(levelname)s %(message)s
+
+[handler_console]
+class = StreamHandler
+stream = sys.stdout
+formatter = generic
+
+[handler_file]
+class = FileHandler
+filename = /home/sugiana/log/mailer.log
+formatter = generic
+
+[logger_root]
+handlers = console, file
+level = INFO

setup.py

 import os
-from setuptools import setup
+from setuptools import (
+    setup,
+    find_packages,
+    )
 
 
 here = os.path.abspath(os.path.dirname(__file__))
@@ -14,6 +17,10 @@ version = line.split()[0]
 
 requires = [
     'sqlalchemy',
+    'zope.sqlalchemy',
+    'psycopg2-binary',
+    'opensipkd-hitung @ '
+    'git+https://git.opensipkd.com/sugiana/opensipkd-hitung.git',
     ]
 
 
@@ -25,7 +32,14 @@ setup(
     author='Owo Sugiana',
     author_email='sugiana@gmail.com',
     license='PostgreSQL License',
-    packages=['imgw'],
+    packages=find_packages(),
+    include_package_data=True,
     install_requires=requires,
     zip_safe=False,
+    entry_points={
+        'console_scripts': [
+            'imgw_init_db = imgw.scripts.init_db:main',
+            'imgw_agent_job = imgw.scripts.agent_job:main',
+            'imgw_mailer = imgw.scripts.mailer:main',
+            ]}
     )