tools.py

import os
import re
import json
import urllib2
import urllib
import csv, codecs, cStringIO
import colander
import locale
import pytz
import io
import random
import sys
import subprocess
from string import ascii_uppercase,ascii_lowercase,digits

from email.utils import parseaddr
from types import (
    IntType,
    LongType,
    )
from datetime import (
    datetime,
    timedelta,
    date
    )

from pyramid.threadlocal import get_current_registry    

STATUS = (
    (1, 'Aktif'),
    (0, 'Inaktif'),
    )    
SUMMARIES = (
    (1, 'Header'),
    (0, 'Detail'),
    )    

################
# Phone number #
################
MSISDN_ALLOW_CHARS = map(lambda x: str(x), range(10)) + ['+']
BULANS = ((1,'Januari'),
          (2,'Februari'),
          (3,'Maret'),
          (4,'April'),
          (5,'Mei'),
          (6,'Juni'),
          (7,'Juli'),
          (8,'Agustus'),
          (9,'September'),
          (10,'Oktober'),
          (11,'November'),
          (12,'Desember'),
          )
          
def email_validator(node, value):
    name, email = parseaddr(value)
    if not email or email.find('@') < 0:
        raise colander.Invalid(node, 'Invalid email format')

def npwpd_validator(npwpd):
    try:
        npwpd = int(npwpd)
        return False
    except ValueError:
        raise colander.Invalid('Invalid NPWPD format')
        return True
        
def get_msisdn(msisdn, country='+62'):
    for ch in msisdn:
        if ch not in MSISDN_ALLOW_CHARS:
            return
    try:
        i = int(msisdn)
    except ValueError, err:
        return
    if not i:
        return
    if len(str(i)) < 7:
        return
    if re.compile(r'^\+').search(msisdn):
        return msisdn
    if re.compile(r'^0').search(msisdn):
        return '%s%s' % (country, msisdn.lstrip('0'))

################
# Money format #
################
def should_int(value):
    int_ = int(value)
    return int_ == value and int_ or value

def thousand(value, float_count=None):
    if float_count is None: # autodetection
        if type(value) in (IntType, LongType):
            float_count = 0
        else:
            float_count = 2
    return locale.format('%%.%df' % float_count, value, True)

def money(value, float_count=None, currency=None):
    if value < 0:
        v = abs(value)
        format_ = '(%s)'
    else:
        v = value
        format_ = '%s'
    if currency is None:
        currency = locale.localeconv()['currency_symbol']
    s = ' '.join([currency, thousand(v, float_count)])
    return format_ % s

###########    
# Pyramid #
###########    
def get_settings():
    return get_current_registry().settings
    
def get_timezone():
    settings = get_settings()
    return pytz.timezone(settings.timezone)

########    
# Time #
########
one_second = timedelta(1.0/24/60/60)
TimeZoneFile = '/etc/timezone'
if os.path.exists(TimeZoneFile):
    DefaultTimeZone = open(TimeZoneFile).read().strip()
else:
    DefaultTimeZone = 'Asia/Jakarta'

def as_timezone(tz_date):
    localtz = get_timezone()
    if not tz_date.tzinfo:
        tz_date = create_datetime(tz_date.year, tz_date.month, tz_date.day,
                                  tz_date.hour, tz_date.minute, tz_date.second,
                                  tz_date.microsecond)
    return tz_date.astimezone(localtz)    

def create_datetime(year, month, day, hour=0, minute=7, second=0,
                     microsecond=0):
    tz = get_timezone()        
    return datetime(year, month, day, hour, minute, second,
                     microsecond, tzinfo=tz)

def create_date(year, month, day):    
    return create_datetime(year, month, day)
    
def create_now():
    tz = get_timezone()
    return datetime.now(tz)
 

def date_from_str(value):
    separator = None
    value = value.split()[0] # dd-mm-yyyy HH:MM:SS  
    for s in ['-', '/']:
        if value.find(s) > -1:
            separator = s
            break    
    if separator:
        t = map(lambda x: int(x), value.split(separator))
        y, m, d = t[2], t[1], t[0]
        if d > 999: # yyyy-mm-dd
            y, d = d, y
    else: # if len(value) == 8: # yyyymmdd
        y, m, d = int(value[:4]), int(value[4:6]), int(value[6:])
    return date(y, m, d)    
    
def dmy(tgl):
    return tgl.strftime('%d-%m-%Y')

def ymd(tgl):
    return tgl.strftime('%Y-%m-%d')

def ymdhms(date):
    if isinstance(date, str):
        return date
    return date.strftime('%Y-%m-%d %H:%M:%S')
    
def dmyhms(date):
    if isinstance(date, str):
        return date
    return date.strftime('%d-%m-%Y %H:%M:%S')
    
def datetime_from_str(values):
    separator = None
    values = values.split()
    value = values[0]  # dd-mm-yyyy HH:MM:SS
        
    tgl = date_from_str(value)
    t = [0,0,0]
    if len(values) > 1:
        t = values[1].split(':')
    return datetime(tgl.year, tgl.month, tgl.day, int(t[0]), int(t[1]), int(t[2]))

def dmy_to_date(tgl):
    return datetime.strptime(tgl, '%d-%m-%Y')
    
def dMy(tgl):
    return str(tgl.day) + ' ' + NAMA_BULAN[tgl.month][1] + ' ' + str(tgl.year)
    
def next_month(year, month):
    if month == 12:
        month = 1
        year += 1
    else:
        month += 1
    return year, month
    
def best_date(year, month, day):
    try:
        return date(year, month, day)
    except ValueError:
        last_day = calendar.monthrange(year, month)[1]
        return date(year, month, last_day)

def next_month_day(year, month, day):
    year, month = next_month(year, month)
    return best_date(year, month, day)
    
################
# Months #
################
BULANS = (
    ('01', 'Januari'),
    ('02', 'Februari'),
    ('03', 'Maret'),
    ('04', 'April'),
    ('05', 'Mei'),
    ('06', 'Juni'),
    ('07', 'Juli'),
    ('08', 'Agustus'),
    ('09', 'September'),
    ('10', 'Oktober'),
    ('11', 'November'),
    ('12', 'Desember'),
    )
    
def get_months(request):
    return BULANS

def email_validator(node, value):
    name, email = parseaddr(value)
    if not email or email.find('@') < 0:
        raise colander.Invalid(node, 'Invalid email format')    
        
def row2dict(row):
    d = {}
    for column in row.__table__.columns:
        d[column.name] = str(getattr(row, column.name))

    return d        
    
def _upper(chain):
    ret = chain.upper()
    if ret:
        return ret
    else:
        return chain
        
    
def clean(s):
    r = ''
    for ch in s:
        if ch not in string.printable:
            ch = ''
        r += ch
    return r

def xls_reader(filename, sheet):    
    workbook = xlrd.open_workbook(filename)
    worksheet = workbook.sheet_by_name(sheet)
    num_rows = worksheet.nrows - 1
    num_cells = worksheet.ncols - 1
    curr_row = -1
    csv = []
    while curr_row < num_rows:
        curr_row += 1
        row = worksheet.row(curr_row)
        curr_cell = -1
        txt = []
        while curr_cell < num_cells:
            curr_cell += 1
            # Cell Types: 0=Empty, 1=Text, 2=Number, 3=Date, 4=Boolean, 5=Error, 6=Blank
            cell_type = worksheet.cell_type(curr_row, curr_cell)
            cell_value = worksheet.cell_value(curr_row, curr_cell)
            if cell_type==1 or cell_type==2:
                try:
                    cell_value = str(cell_value)
                except:
                    cell_value = '0'
            else:
                cell_value = clean(cell_value)
                
            if curr_cell==0 and cell_value.strip()=="Tanggal":
                curr_cell=num_cells
            elif curr_cell==0 and cell_value.strip()=="":
                curr_cell = num_cells
                curr_row = num_rows
            else:
                txt.append(cell_value)
        if txt:
            csv.append(txt)
    return csv        


class UTF8Recoder:
    """
    Iterator that reads an encoded stream and reencodes the input to UTF-8
    """
    def __init__(self, f, encoding):
        self.reader = codecs.getreader(encoding)(f)

    def __iter__(self):
        return self

    def next(self):
        return self.reader.next().encode("utf-8")

class UnicodeReader:
    """
    A CSV reader which will iterate over lines in the CSV file "f",
    which is encoded in the given encoding.
    """

    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
        f = UTF8Recoder(f, encoding)
        self.reader = csv.reader(f, dialect=dialect, **kwds)

    def next(self):
        row = self.reader.next()
        return [unicode(s, "utf-8") for s in row]

    def __iter__(self):
        return self

class UnicodeWriter:
    """
    A CSV writer which will write rows to CSV file "f",
    which is encoded in the given encoding.
    """

    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
        # Redirect output to a queue
        self.queue = cStringIO.StringIO()
        self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
        self.stream = f
        self.encoder = codecs.getincrementalencoder(encoding)()

    def writerow(self, row):
        self.writer.writerow([s.encode("utf-8") for s in row])
        # Fetch UTF-8 output from the queue ...
        data = self.queue.getvalue()
        data = data.decode("utf-8")
        # ... and reencode it into the target encoding
        data = self.encoder.encode(data)
        # write to the target stream
        print data
        self.stream.write(data)
        # empty queue
        self.queue.truncate(0)

    def writerows(self, rows):
        for row in rows:
            self.writerow(row)
            
class CSVRenderer(object):
   def __init__(self, info):
      pass

   def __call__(self, value, system):
      """ Returns a plain CSV-encoded string with content-type
      ``text/csv``. The content-type may be overridden by
      setting ``request.response.content_type``."""

      request = system.get('request')
      if request is not None:
         response = request.response
         ct = response.content_type
         if ct == response.default_content_type:
            response.content_type = 'text/csv'

      fout = io.BytesIO() #StringIO()
      fcsv = csv.writer(fout, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
      #fcsv = UnicodeWriter(fout, delimiter=',', quotechar=',', quoting=csv.QUOTE_MINIMAL)
      #print value.get('header', [])
      fcsv.writerow(value.get('header', []))
      fcsv.writerows(value.get('rows', []))

      return fout.getvalue()    

########    
# File #
########    
# http://stackoverflow.com/questions/2257441/random-string-generation-with-upper-case-letters-and-digits-in-python
def get_random_string(width=6):
    return ''.join(random.choice(ascii_uppercase + ascii_lowercase + digits) \
        for _ in range(width))
        
def get_ext(filename):
    return os.path.splitext(filename)[-1]
    
def file_type(filename):    
    ctype, encoding = mimetypes.guess_type(filename)
    if ctype is None or encoding is not None:
        ctype = 'application/octet-stream'
    return ctype    

class SaveFile(object):
    def __init__(self, dir_path):
        self.dir_path = dir_path
        
    def create_fullpath(self, ext=''):
        while True:
            filename = get_random_string() + ext
            fullpath = os.path.join(self.dir_path, filename)
            if not os.path.exists(fullpath):
                return fullpath
        
    def save(self, content, filename=None):
        fullpath = create_fullpath()
        f = open(fullpath, 'wb')
        f.write(content)
        f.close()
        return fullpath
        
class Upload(SaveFile):
    def save(self, request, name):
        input_file = request.POST[name].file
        ext = get_ext(request.POST[name].filename)
        fullpath = self.create_fullpath(ext)
        output_file = open(fullpath, 'wb')
        input_file.seek(0)
        while True:
            data = input_file.read(2<<16)
            if not data:
                break
            output_file.write(data)
        output_file.close()
        return fullpath

class UploadFiles(SaveFile):
    def save(self, fs):
        input_file = fs.file
        ext = get_ext(fs.filename)
        fullpath = self.create_fullpath(ext)
        output_file = open(fullpath, 'wb')
        input_file.seek(0)
        while True:
            data = input_file.read(2<<16)
            if not data:
                break
            output_file.write(data)
        output_file.close()
        return fullpath  
        
def to_str(v):
    typ = type(v)
    print typ, v
    if typ == DateType:
        return dmy(v)
    if typ == DateTimeType:
        return dmyhms(v)
    if v == 0:
        return '0'
    if typ in [UnicodeType, StringType]:
        return v.strip()
    elif typ is BooleanType:
        return v and '1' or '0'
    return v and str(v) or ''
    
def dict_to_str(d):
    r = {}
    for key in d:
        val = d[key]        
        r[key] = to_str(val)
    return r        
    
# Data Tables
def _DTstrftime(chain):
    ret = chain and datetime.strftime(chain, "%d-%m-%Y")
    if ret:
      return ret
    else:
      return chain
      
def _DTnumberformat(chain):
    import locale
    locale.setlocale(locale.LC_ALL, get_settings()['localization'])
    ret = locale.format("%d", chain, grouping=True)
    if ret:
      return ret
    else:
      return chain
      
def _DTactive(chain):
    ret = chain==1 and 'Aktif' or 'Inaktif'
    if ret:
      return ret
    else:
      return chain

      
#Captcha Response
class RecaptchaResponse(object):
    def __init__(self, is_valid, error_code=None):
        self.is_valid = is_valid
        self.error_code = error_code

def captcha_submit(recaptcha_challenge_field,
            recaptcha_response_field,
            private_key,
            remoteip):
    """
    Submits a reCAPTCHA request for verification. Returns RecaptchaResponse
    for the request

    recaptcha_challenge_field -- The value of recaptcha_challenge_field from the form
    recaptcha_response_field -- The value of recaptcha_response_field from the form
    private_key -- your reCAPTCHA private key
    remoteip -- the user's ip address
    """

    if not (recaptcha_response_field and recaptcha_challenge_field and
            len (recaptcha_response_field) and len (recaptcha_challenge_field)):

        return RecaptchaResponse (is_valid = False, error_code = 'incorrect-captcha-sol')


    def encode_if_necessary(s):
        if isinstance(s, unicode):
            return s.encode('utf-8')
        return s

    params = urllib.urlencode ({
            'privatekey':  encode_if_necessary(private_key),
            'remoteip'  :  encode_if_necessary(remoteip),
            'secret'    :  encode_if_necessary(recaptcha_challenge_field),
            'response'  :  encode_if_necessary(recaptcha_response_field),
            })
            
    #print "https://%s/recaptcha/api/siteverify" % VERIFY_SERVER
    request = urllib2.Request (
        url = "https://www.google.com/recaptcha/api/siteverify",
        data = params,
        headers = {
            "Content-type": "application/x-www-form-urlencoded",
            "User-agent": "reCAPTCHA Python"
            }
        )
    httpresp = urllib2.urlopen (request)

    return_values = json.loads(httpresp.read())
    httpresp.close()
    print return_values
    return_code = return_values['success']
    if (return_code == True):
        return RecaptchaResponse (is_valid=True)
    else:
        return RecaptchaResponse (is_valid=False, error_code = return_values['error-codes'])

# Multi-Dict dihasilkan oleh request.POST.items(). Widget yang bertipe Date
# semua name berisi "date" pada HTML source karena diperlakukan sebagai list.
# Jadi perlu fungsi berikut ini untuk menerjemahkannya menjadi dictionary
# biasa.
# request.POST.items() menghasilkan:
# [('__start__', u'tgl_tetap:mapping'), ('date', u'2018-07-31'), ('__end__', u'tgl_tetap:mapping')]
def multi_dict_values(md):
    d = dict()
    real_key = None
    for key, value in md:
        if key == '__start__':
            real_key = value.split(':')[0]
        elif real_key:
            d[real_key] = value
            real_key = None
        elif key == '__end__':
            pass
        else:
            d[key] = value
    return d

def csv_response(request, value, filename):
    response = request.response
    response.content_type = 'text/csv'
    # response.content_disposition = 'attachment;filename=' + filename
    response.content_disposition = 'filename=' + filename
    if sys.version_info < (3,):
        import StringIO
        fout = StringIO.StringIO()
    else:
        fout = io.StringIO()

    fcsv = csv.writer(fout, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
    fcsv.writerow(value.get('header', []))
    fcsv.writerows(value.get('rows', []))
    response.write(fout.getvalue())
    return response

def odt_export(request, filename, file_type):
    results = odt_export_(filename, file_type)
    
    if 'error' in results:
        return results
    out_filename = results['filename']
    if not os.path.isfile(out_filename):
        return dict(error=True,
                    msg='Error %s tidak ditemukan ' % out_filename)

    with open(out_filename, 'rb') as f:
        return file_response(request, f, out_filename, file_type)

def odt_export_(filename, file_type, password=None):
    settings = get_settings()
    import getpass
    username = getpass.getuser()

    odt_file = '.'.join([filename, 'odt'])
    out_dir = os.path.dirname(filename)
    if 'unoconv_py' in settings and settings['unoconv_py']:
        unoconv_py = settings['unoconv_py']
        if 'unoconv_bin' in settings and settings['unoconv_bin']:
            unoconv_bin = settings['unoconv_bin']
        else:
            unoconv_bin = ''

        params = [unoconv_py, unoconv_bin, '-f', file_type]
        if password:
            params.extend(['-e', 'EncryptFile=True',
                           '-e', 'DocumentOpenPassword='+password])

        params.append(odt_file)
        print("DEBUG EXPORT>>", ' '.join(params))

        subprocess.call(params)

    # convert using bin
    else:
        if 'unoconv_bin' in settings and settings['unoconv_bin']:
            unoconv_bin = settings['unoconv_bin']
            params = [unoconv_bin,
                      '-env:UserInstallation=file:///tmp/'+username,
                      '--headless', '--convert-to', 'pdf']
            params.extend(['--outdir', out_dir, file_type, odt_file])
            print("DEBUG EXPORT>>", ' '.join(params))
            subprocess.call(params)

    out_file = '.'.join([filename, file_type])
    if not os.path.isfile(odt_file):
        return dict(error=dict(code=-1,
                               message='File  %s tidak ditemukan ' % odt_file))
    else:
        if not os.path.isfile(out_file):
            return dict(error=dict(code=-1,
                                   message='File  %s tidak ditemukan ' % out_file))
        print("DEBUG DELETE>>", odt_file)
        os.remove(odt_file)
    print("DEBUG OUT FILE>>", out_file)
    return dict(filename=out_file)
    
def file_response(request, f=None, filename=None, filetype=None):
    """
    :param request:
    :param f:  object file
    :param filename:
    :param filetype: type of file
    :return: object response
    """
    import ntpath
    if not f:
        f = open(filename, 'rb')
        fname = ntpath.basename(filename)
    else:
        fname = filename

    if not filetype:
        t = fname.split('.')
        filetype = ''.join(t[-1:])

    response = request.response
    response.content_type = "application/" + filetype
    response.content_disposition = 'filename=' + fname
    response.write(f.read())
    return response
    
def terbilang(bil):
    angka = ["", "Satu", "Dua", "Tiga", "Empat", "Lima", "Enam", "Tujuh", "Delapan", "Sembilan", "Sepuluh", "Sebelas"]
    hasil = " "
    n = int(bil)
    if n >= 0 and n <= 11:
        hasil = hasil + angka[n]
    elif n < 20:
        hasil = terbilang(n % 10) + " Belas"
    elif n < 100:
        hasil = terbilang(n / 10) + " Puluh" + terbilang(n % 10)
    elif n < 200:
        hasil = " Seratus" + terbilang(n - 100)
    elif n < 1000:
        hasil = terbilang(n / 100) + " Ratus" + terbilang(n % 100)
    elif n < 2000:
        hasil = " Seribu" + terbilang(n - 1000)
    elif n < 1000000:
        hasil = terbilang(n / 1000) + " Ribu" + terbilang(n % 1000)
    elif n < 1000000000:
        hasil = terbilang(n / 1000000) + " Juta" + terbilang(n % 1000000)
    else:
        hasil = terbilang(n / 1000000000) + " Miliar" + terbilang(n % 1000000000)
    return hasil
    
def int_to_roman(num):
    val = [
        1000, 900, 500, 400,
        100, 90, 50, 40,
        10, 9, 5, 4,
        1
        ]
    syb = [
        "M", "CM", "D", "CD",
        "C", "XC", "L", "XL",
        "X", "IX", "V", "IV",
        "I"
        ]
    roman_num = ''
    i = 0
    while  num > 0:
        for _ in range(num // val[i]):
            roman_num += syb[i]
            num -= val[i]
        i += 1
    return roman_num
    
def round_up(value):
    if isinstance(value, str):
        value = float(int(value))
    if isinstance(value, int):
        value = float(value)
    return int(round(value+0.4999))
    
def get_tmp():
    settings = get_settings()
    if 'tmp_files' in settings and settings['tmp_files']:
        tmpf = settings['tmp_files'].strip('/')
        return str(tmpf)+'/'
    else:
        return str("/tmp/")