Merge pull request #28 from atf1206/update_qpython

Update to qPython 2.0 and add support for utf-8

QCon.py

@@ -15,7 +15,7 @@ def __init__(self, host, port, username, password, name=None, hdb=False):
         self.port = port
         self.username = username
         self.password = password
-        self.q = qconnection.QConnection(host = self.host, port = self.getPort(), username = self.username, password = self.password)
+        self.q = qconnection.QConnection(host = self.host, port = self.getPort(), username = self.username, password = self.password, encoding = 'UTF-8')
 
     @classmethod
     def fromH(cls, h):

qpython/__init__.py

@@ -17,7 +17,7 @@
 __all__ = ['qconnection', 'qtype', 'qtemporal', 'qcollection']
 
 
-__version__ = '1.1.0'
+__version__ = '2.0.0'
 
 
 
@@ -66,4 +66,4 @@ def union_dict(self, **kw):
                               numpy_temporals = False,
                               pandas = False,
                               single_char_strings = False
-                             )
+                             )

qpython/_pandas.py

@@ -22,7 +22,7 @@
 
 from collections import OrderedDict
 
-from  import MetaData
+from . import MetaData
 from .qreader import QReader, QReaderException
 from .qcollection import QDictionary, qlist
 from .qwriter import QWriter, QWriterException
@@ -32,7 +32,8 @@
 
 class PandasQReader(QReader):
 
-    parse = Mapper(QReader._reader_map)
+    _reader_map = dict.copy(QReader._reader_map)
+    parse = Mapper(_reader_map)
 
     @parse(QDICTIONARY)
     def _read_dictionary(self, qtype = QDICTIONARY):
@@ -57,8 +58,8 @@ def _read_dictionary(self, qtype = QDICTIONARY):
 
                 return table
             else:
-                keys = keys if not isinstance(keys, pandas.Series) else keys.as_matrix()
-                values = values if not isinstance(values, pandas.Series) else values.as_matrix()
+                keys = keys if not isinstance(keys, pandas.Series) else keys.values
+                values = values if not isinstance(values, pandas.Series) else values.values
                 return QDictionary(keys, values)
         else:
             return QReader._read_dictionary(self, qtype = qtype)
@@ -71,7 +72,8 @@ def _read_table(self, qtype = QTABLE):
             self._buffer.skip()  # ignore dict type stamp
 
             columns = self._read_object()
-            data = self._read_object()
+            self._buffer.skip() # ignore generic list type indicator
+            data = QReader._read_general_list(self, qtype)
 
             odict = OrderedDict()
             meta = MetaData(qtype = QTABLE)
@@ -106,34 +108,36 @@ def _read_list(self, qtype):
         if self._options.pandas:
             self._options.numpy_temporals = True
 
-        list = QReader._read_list(self, qtype = qtype)
+        qlist = QReader._read_list(self, qtype = qtype)
 
         if self._options.pandas:
             if -abs(qtype) not in [QMONTH, QDATE, QDATETIME, QMINUTE, QSECOND, QTIME, QTIMESTAMP, QTIMESPAN, QSYMBOL]:
                 null = QNULLMAP[-abs(qtype)][1]
-                ps = pandas.Series(data = list).replace(null, numpy.NaN)
+                ps = pandas.Series(data = qlist).replace(null, numpy.NaN)
             else:
-                ps = pandas.Series(data = list)
+                ps = pandas.Series(data = qlist)
 
             ps.meta = MetaData(qtype = qtype)
             return ps
         else:
-            return list
+            return qlist
 
 
     @parse(QGENERAL_LIST)
     def _read_general_list(self, qtype = QGENERAL_LIST):
-        list = QReader._read_general_list(self, qtype)
+        qlist = QReader._read_general_list(self, qtype)
         if self._options.pandas:
-            return [numpy.nan if isinstance(element, basestring) and element == b' ' else element for element in list]
+            return [numpy.nan if isinstance(element, basestring) and element == b' ' else element for element in qlist]
         else:
-            return list
+            return qlist
 
 
 
 class PandasQWriter(QWriter):
 
-    serialize = Mapper(QWriter._writer_map)
+    _writer_map = dict.copy(QWriter._writer_map)
+    serialize = Mapper(_writer_map)
+
 
     @serialize(pandas.Series)
     def _write_pandas_series(self, data, qtype = None):
@@ -164,19 +168,19 @@ def _write_pandas_series(self, data, qtype = None):
             raise QWriterException('Unable to serialize pandas series %s' % data)
 
         if qtype == QGENERAL_LIST:
-            self._write_generic_list(data.as_matrix())
+            self._write_generic_list(data.values)
         elif qtype == QCHAR:
-            self._write_string(data.replace(numpy.nan, ' ').as_matrix().astype(numpy.string_).tostring())
+            self._write_string(data.replace(numpy.nan, ' ').values.astype(numpy.string_).tostring())
         elif data.dtype.type not in (numpy.datetime64, numpy.timedelta64):
             data = data.fillna(QNULLMAP[-abs(qtype)][1])
-            data = data.as_matrix()
+            data = data.values
 
             if PY_TYPE[qtype] != data.dtype:
                 data = data.astype(PY_TYPE[qtype])
 
             self._write_list(data, qtype = qtype)
         else:
-            data = data.as_matrix()
+            data = data.values
             data = data.astype(TEMPORAL_Q_TYPE[qtype])
             self._write_list(data, qtype = qtype)
 
@@ -207,7 +211,10 @@ def _write_pandas_data_frame(self, data, qtype = None):
 
     @serialize(tuple, list)
     def _write_generic_list(self, data):
-        self._buffer.write(struct.pack('=bxi', QGENERAL_LIST, len(data)))
-        for element in data:
-            # assume nan represents a string null
-            self._write(' ' if type(element) in [float, numpy.float32, numpy.float64] and numpy.isnan(element) else element)
+        if self._options.pandas:
+            self._buffer.write(struct.pack('=bxi', QGENERAL_LIST, len(data)))
+            for element in data:
+                # assume nan represents a string null
+                self._write(' ' if type(element) in [float, numpy.float32, numpy.float64] and numpy.isnan(element) else element)
+        else:
+            QWriter._write_generic_list(self, data)

qpython/qconnection.py

@@ -62,6 +62,9 @@ class QConnection(object):
      - `username` (`string` or `None`) - username for q authentication/authorization
      - `password` (`string` or `None`) - password for q authentication/authorization
      - `timeout` (`nonnegative float` or `None`) - set a timeout on blocking socket operations
+     - `encoding` (`string`) - string encoding for data deserialization
+     - `reader_class` (subclass of `QReader`) - data deserializer
+     - `writer_class` (subclass of `QWriter`) - data serializer
     :Options: 
      - `raw` (`boolean`) - if ``True`` returns raw data chunk instead of parsed 
        data, **Default**: ``False``
@@ -74,19 +77,37 @@ class QConnection(object):
        strings are encoded as q strings instead of chars, **Default**: ``False``
     '''
 
-    def __init__(self, host, port, username = None, password = None, timeout = None, **options):
+
+    def __init__(self, host, port, username = None, password = None, timeout = None, encoding = 'latin-1', reader_class = None, writer_class = None, **options):
         self.host = host
         self.port = port
         self.username = username
         self.password = password
 
         self._connection = None
+        self._connection_file = None
         self._protocol_version = None
 
         self.timeout = timeout
 
+        self._encoding = encoding
+
         self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options))
 
+        try:
+            from qpython._pandas import PandasQReader, PandasQWriter
+            self._reader_class = PandasQReader
+            self._writer_class = PandasQWriter
+        except ImportError:
+            self._reader_class = QReader
+            self._writer_class = QWriter
+
+        if reader_class:
+            self._reader_class = reader_class
+
+        if writer_class:
+            self._writer_class = writer_class
+
 
     def __enter__(self):
         self.open()
@@ -122,8 +143,8 @@ def open(self):
             self._init_socket()
             self._initialize()
 
-            self._writer = QWriter(self._connection, protocol_version = self._protocol_version)
-            self._reader = QReader(self._connection.makefile('b'))
+            self._writer = self._writer_class(self._connection, protocol_version = self._protocol_version, encoding = self._encoding)
+            self._reader = self._reader_class(self._connection_file, encoding = self._encoding)
 
 
     def _init_socket(self):
@@ -132,14 +153,18 @@ def _init_socket(self):
             self._connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
             self._connection.connect((self.host, self.port))
             self._connection.settimeout(self.timeout)
+            self._connection_file = self._connection.makefile('b')
         except:
             self._connection = None
+            self._connection_file = None
             raise
 
 
     def close(self):
         '''Closes connection with the q service.'''
         if self._connection:
+            self._connection_file.close()
+            self._connection_file = None
             self._connection.close()
             self._connection = None
 
@@ -160,7 +185,7 @@ def is_connected(self):
     def _initialize(self):
         '''Performs a IPC protocol handshake.'''
         credentials = (self.username if self.username else '') + ':' + (self.password if self.password else '')
-        credentials = credentials.encode('latin-1')
+        credentials = credentials.encode(self._encoding)
         self._connection.send(credentials + b'\3\0')
         response = self._connection.recv(1)
 
@@ -220,7 +245,7 @@ def query(self, msg_type, query, *parameters, **options):
             self._writer.write([query] + list(parameters), msg_type, **self._options.union_dict(**options))
 
 
-    def sync(self, query, *parameters, **options):
+    def sendSync(self, query, *parameters, **options):
         '''Performs a synchronous query against a q service and returns parsed 
         data.
         
@@ -230,23 +255,23 @@ def sync(self, query, *parameters, **options):
         
         Executes a q expression:
         
-            >>> print(q.sync('til 10'))
+            >>> print(q.sendSync('til 10'))
             [0 1 2 3 4 5 6 7 8 9]
         
         Executes an anonymous q function with a single parameter:
         
-            >>> print(q.sync('{til x}', 10))
+            >>> print(q.sendSync('{til x}', 10))
             [0 1 2 3 4 5 6 7 8 9]
             
         Executes an anonymous q function with two parameters:
         
-            >>> print(q.sync('{y + til x}', 10, 1))
+            >>> print(q.sendSync('{y + til x}', 10, 1))
             [ 1  2  3  4  5  6  7  8  9 10]
             
-            >>> print(q.sync('{y + til x}', *[10, 1]))
+            >>> print(q.sendSync('{y + til x}', *[10, 1]))
             [ 1  2  3  4  5  6  7  8  9 10]
         
-        The :func:`.sync` is called from the overloaded :func:`.__call__` 
+        The :func:`.sendSync` is called from the overloaded :func:`.__call__`
         function. This allows :class:`.QConnection` instance to be called as 
         a function:
         
@@ -284,7 +309,7 @@ def sync(self, query, *parameters, **options):
             raise QReaderException('Received message of type: %s where response was expected')
 
 
-    def async(self, query, *parameters, **options):
+    def sendAsync(self, query, *parameters, **options):
         '''Performs an asynchronous query and returns **without** retrieving of 
         the response.
         
@@ -294,11 +319,11 @@ def async(self, query, *parameters, **options):
         
         Calls a anonymous function with a single parameter:
         
-            >>> q.async('{til x}', 10)
+            >>> q.sendAsync('{til x}', 10)
         
         Executes a q expression:
         
-            >>> q.async('til 10')
+            >>> q.sendAsync('til 10')
         
         :Parameters:
          - `query` (`string`) - query to be executed
@@ -357,4 +382,4 @@ def receive(self, data_only = True, **options):
 
 
     def __call__(self, *parameters, **options):
-        return self.sync(parameters[0], *parameters[1:], **options)
+        return self.sendSync(parameters[0], *parameters[1:], **options)

qpython/qreader.py

@@ -99,28 +99,21 @@ class QReader(object):
     
     :Parameters:
      - `stream` (`file object` or `None`) - data input stream
+     - `encoding` (`string`) - encoding for characters parsing
+
+    :Attrbutes:
+     - `_reader_map` - stores mapping between q types and functions
+       responsible for parsing into Python objects
     '''
 
     _reader_map = {}
     parse = Mapper(_reader_map)
 
 
-    def __new__(cls, *args, **kwargs):
-        if cls is QReader:
-            # try to load optional pandas binding
-            try:
-                from qpython._pandas import PandasQReader
-                return super(QReader, cls).__new__(PandasQReader)
-            except ImportError:
-                return super(QReader, cls).__new__(QReader)
-        else:
-            #return super(QReader, cls).__new__(cls)
-            return super().__new__(cls) #komsit fix
-
-
-    def __init__(self, stream):
+    def __init__(self, stream, encoding = 'latin-1'):
         self._stream = stream
         self._buffer = QReader.BytesBuffer()
+        self._encoding = encoding
 
 
     def read(self, source = None, **options):
@@ -207,7 +200,7 @@ def read_data(self, message_size, is_compressed = False, **options):
             uncompressed_size = -8 + self._buffer.get_int()
             compressed_data = self._read_bytes(message_size - 12) if self._stream else self._buffer.raw(message_size - 12)
 
-            raw_data = numpy.fromstring(compressed_data, dtype = numpy.uint8)
+            raw_data = numpy.frombuffer(compressed_data, dtype = numpy.uint8)
             if  uncompressed_size <= 0:
                 raise QReaderException('Error while data decompression.')
 
@@ -226,7 +219,7 @@ def read_data(self, message_size, is_compressed = False, **options):
     def _read_object(self):
         qtype = self._buffer.get_byte()
 
-        reader = QReader._reader_map.get(qtype, None)
+        reader = self._get_reader(qtype)
 
         if reader:
             return reader(self, qtype)
@@ -238,6 +231,10 @@ def _read_object(self):
         raise QReaderException('Unable to deserialize q type: %s' % hex(qtype))
 
 
+    def _get_reader(self, qtype):
+        return self._reader_map.get(qtype, None)
+
+
     @parse(QERROR)
     def _read_error(self, qtype = QERROR):
         raise QException(self._read_symbol())
@@ -257,7 +254,7 @@ def _read_symbol(self, qtype = QSYMBOL):
 
     @parse(QCHAR)
     def _read_char(self, qtype = QCHAR):
-        return chr(self._read_atom(QCHAR)).encode('latin-1') 
+        return chr(self._read_atom(QCHAR)).encode(self._encoding)
 
 
     @parse(QGUID)
@@ -299,7 +296,7 @@ def _read_list(self, qtype):
             return qlist(data, qtype = qtype, adjust_dtype = False)
         elif conversion:
             raw = self._buffer.raw(length * ATOM_SIZE[qtype])
-            data = numpy.fromstring(raw, dtype = conversion)
+            data = numpy.frombuffer(raw, dtype = conversion)
             if not self._is_native:
                 data.byteswap(True)
 

qpython/qtype.py

@@ -259,8 +259,8 @@
 _QNULL2 = numpy.int16(-2**15)
 _QNULL4 = numpy.int32(-2**31)
 _QNULL8 = numpy.int64(-2**63)
-_QNAN32 = numpy.fromstring(b'\x00\x00\xc0\x7f', dtype=numpy.float32)[0]
-_QNAN64 = numpy.fromstring(b'\x00\x00\x00\x00\x00\x00\xf8\x7f', dtype=numpy.float64)[0]
+_QNAN32 = numpy.frombuffer(b'\x00\x00\xc0\x7f', dtype=numpy.float32)[0]
+_QNAN64 = numpy.frombuffer(b'\x00\x00\x00\x00\x00\x00\xf8\x7f', dtype=numpy.float64)[0]
 _QNULL_BOOL = numpy.bool_(False)
 _QNULL_SYM = numpy.string_('')
 _QNULL_GUID = uuid.UUID('00000000-0000-0000-0000-000000000000')