I have a library in c++ and I'm trying to wrap it for python using Cython. One function returns an array of 3D vectors (float (*x)[3]) and I want to access that data from python. I was able to do so by doing something like
res = [
(self.thisptr.x[j][0],self.thisptr.x[j][1],self.thisptr.x[j][2])
for j in xrange(self.natoms)
]
but I would like to access this as a numpy array, so I tried numpy.array on that and it was much slower. I also tried
cdef np.ndarray res = np.zeros([self.thisptr.natoms,3], dtype=np.float)
cdef int i
for i in range(self.natoms):
res[i][0] = self.thisptr.x[i][0]
res[i][1] = self.thisptr.x[i][1]
res[i][2] = self.thisptr.x[i][2]
But is about three times slower than the list version.
Any suggestions on how to convert the list of vectors to an numpy array faster?
The complete code is
cimport cython
import numpy as np
cimport numpy as np
ctypedef np.float_t ftype_t
cdef extern from "ccxtc.h" namespace "ccxtc":
cdef cppclass xtc:
xtc(char []) except +
int next()
int natoms
float (*x)[3]
float time
cdef class pyxtc:
cdef xtc *thisptr
def __cinit__(self, char fname[]):
self.thisptr = new xtc(fname)
def __dealloc__(self):
del self.thisptr
property natoms:
def __get__(self):
return self.thisptr.natoms
property x:
def __get__(self):
cdef np.ndarray res = np.zeros([self.thisptr.natoms,3], dtype=np.float)
cdef int i
for i in range(self.natoms):
res[i][0] = self.thisptr.x[i][0]
res[i][1] = self.thisptr.x[i][1]
res[i][2] = self.thisptr.x[i][2]
return res
#return [ (self.thisptr.x[j][0],self.thisptr.x[j][1],self.thisptr.x[j][2]) for j in xrange(self.natoms)]
#cython.boundscheck(False)
def next(self):
return self.thisptr.next()
Define the type of res:
cdef np.ndarray[np.float64_t, ndim=2] res = ...
Use full index:
res[i,0] = ...
Turn off boundscheck & wraparound
#cython.boundscheck(False)
#cython.wraparound(False)
To summarize what HYRY said and to ensure Cython can generate fast indexing code, try something like the following:
cimport cython
import numpy as np
cimport numpy as np
ctypedef np.float_t ftype_t
cdef extern from "ccxtc.h" namespace "ccxtc":
cdef cppclass xtc:
xtc(char []) except +
int next()
int natoms
float (*x)[3]
float time
cdef class pyxtc:
cdef xtc *thisptr
def __cinit__(self, char fname[]):
self.thisptr = new xtc(fname)
def __dealloc__(self):
del self.thisptr
property natoms:
def __get__(self):
return self.thisptr.natoms
#cython.boundscheck(False)
#cython.wraparound(False)
cdef _ndarray_from_x(self):
cdef np.ndarray[np.float_t, ndim=2] res = np.zeros([self.thisptr.natoms,3], dtype=np.float)
cdef int i
for i in range(self.thisptr.natoms):
res[i,0] = self.thisptr.x[i][0]
res[i,1] = self.thisptr.x[i][1]
res[i,2] = self.thisptr.x[i][2]
return res
property x:
def __get__(self):
return self._ndarray_from_x()
#cython.boundscheck(False)
def next(self):
return self.thisptr.next()
All I did was put the fast stuff inside a cdef method, put the right optimizing decorators on it, and called that inside the property's __get__(). You should also make sure to refer to self.thisptr.natoms inside the range() call rather than use the natoms property, which has lots of Python overhead associated with it.
Related
I am trying to wrap to python3 a very simple Object Oriented C-API. This basically follow the same pattern as:
Developing C wrapper API for Object-Oriented C++ code or,
Wrapping C++ class API for C consumption
So basically my issue is as follow, I have the following input API:
$ cat api.h
/* ns::a */
typedef struct ns_a ns_a;
ns_a* ns_a_create();
void ns_a_delete(ns_a* a);
bool ns_a_execute(ns_a* a);
/* ns::b */
typedef struct ns_b ns_b;
ns_b* ns_b_create();
void ns_b_delete(ns_b* b);
bool ns_b_run(ns_b* b);
Which I then wrap using the following steps:
class _TypeSafeHandle:
def __init__(self, ptr):
self._as_parameter_ = ptr
#classmethod
def from_param(cls, obj):
if obj.__class__ != cls:
raise ValueError(f"Not a {obj.__class__.__name__} reference")
if not obj._as_parameter_:
raise ValueError("Passing undefined instance")
return obj
#final
class _A(_TypeSafeHandle):
""" for library author, not intended to be exposed publicly """
def __init__(self, ptr):
super().__init__(ptr)
#final
class _B(_TypeSafeHandle):
def __init__(self, ptr):
super().__init__(ptr)
Given the type definition I can now expose the C-API as:
# a
ns_a_create = _func('ns_a_create', c_void_p, None, _check_ns_a)
ns_a_delete = _func('ns_a_delete', None, [_A])
ns_a_execute = _func('ns_a_execute', c_bool, [_A])
# b
ns_b_create = _func('ns_b_create', c_void_p, None, _check_ns_b)
ns_b_delete = _func('ns_b_delete', None, [_B])
ns_b_run = _func('ns_b_run', c_bool, [_B])
Using:
def _func(name, restype, argtypes, errcheck):
func = getattr(_lib, name)
func.restype = restype
func.argtypes = argtypes
func.errcheck = errcheck
return func
and:
def _check_ns_a(result, _func, _args):
if result is None:
raise MemoryError("internal memory allocation failure")
if type(result) != int:
raise TypeError("Expecting int value for pointer")
return = _A(c_void_p(result))
and
def _check_ns_b(result, _func, _args):
if result is None:
raise MemoryError("internal memory allocation failure")
if type(result) != int:
raise TypeError("Expecting int value for pointer")
return = _B(c_void_p(result))
At this step, I can now expose the class A to my user:
class A:
def __init__(self):
self._a = ns_a_create()
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
ns_a_delete(self._a)
def execute(self):
ret = ns_a_execute(self._a)
if not ret:
raise ValueError("Could not execute")
I'd like to keep the intermediate _A and _B class as implementation detail for the library author (avoid shooting in the foot passing wrong instance to wrong delete function). I find it more readable than declaring a forward declared class A:
class A: # forward declaration
pass
# _func declaration with type `A` instead of `_A`
class A:
# actual implementation detail
So my question is: is there a way to reduce code redundancy in between _A and _B ? Is there a way to reduce code redundancy in between _check_ns_a and _check_ns_b ?
I'd like to have several ways of constructing a cdef class in Cython, but I can't figure out how to use factory functions with extension classes that would be exportable to other modules using .pxd files.
The Cython documentation has the following example with factory functions, which works as expected when I compile it:
from libc.stdlib cimport malloc, free
# Example C struct
ctypedef struct my_c_struct:
int a
int b
cdef class WrapperClass:
"""A wrapper class for a C/C++ data structure"""
cdef my_c_struct *_ptr
cdef bint ptr_owner
def __cinit__(self):
self.ptr_owner = False
def __dealloc__(self):
# De-allocate if not null and flag is set
if self._ptr is not NULL and self.ptr_owner is True:
free(self._ptr)
self._ptr = NULL
# Extension class properties
#property
def a(self):
return self._ptr.a if self._ptr is not NULL else None
#property
def b(self):
return self._ptr.b if self._ptr is not NULL else None
#staticmethod
cdef WrapperClass from_ptr(my_c_struct *_ptr, bint owner=False):
"""Factory function to create WrapperClass objects from
given my_c_struct pointer.
Setting ``owner`` flag to ``True`` causes
the extension type to ``free`` the structure pointed to by ``_ptr``
when the wrapper object is deallocated."""
# Call to __new__ bypasses __init__ constructor
cdef WrapperClass wrapper = WrapperClass.__new__(WrapperClass)
wrapper._ptr = _ptr
wrapper.ptr_owner = owner
return wrapper
#staticmethod
cdef WrapperClass new_struct():
"""Factory function to create WrapperClass objects with
newly allocated my_c_struct"""
cdef my_c_struct *_ptr = <my_c_struct *>malloc(sizeof(my_c_struct))
if _ptr is NULL:
raise MemoryError
_ptr.a = 0
_ptr.b = 0
return WrapperClass.from_ptr(_ptr, owner=True)
However, when I try to break out the cdef declarations into a .pxd file like this:
WrapperClass.pxd
ctypedef struct my_c_struct:
int a
int b
cdef class WrapperClass:
cdef my_c_struct *_ptr
cdef bint ptr_owner
cdef WrapperClass from_ptr(my_c_struct *_ptr, bint owner)
cdef WrapperClass new_struct()
WrapperClass.pyx
# cython: language_level=3
from libc.stdlib cimport malloc, free
cdef class WrapperClass:
"""A wrapper class for a C/C++ data structure"""
def __cinit__(self):
self.ptr_owner = False
def __dealloc__(self):
# De-allocate if not null and flag is set
if self._ptr is not NULL and self.ptr_owner is True:
free(self._ptr)
self._ptr = NULL
# Extension class properties
#property
def a(self):
return self._ptr.a if self._ptr is not NULL else None
#property
def b(self):
return self._ptr.b if self._ptr is not NULL else None
#staticmethod
cdef WrapperClass from_ptr(my_c_struct *_ptr, bint owner=False):
"""Factory function to create WrapperClass objects from
given my_c_struct pointer.
Setting ``owner`` flag to ``True`` causes
the extension type to ``free`` the structure pointed to by ``_ptr``
when the wrapper object is deallocated."""
# Call to __new__ bypasses __init__ constructor
cdef WrapperClass wrapper = WrapperClass.__new__(WrapperClass)
wrapper._ptr = _ptr
wrapper.ptr_owner = owner
return wrapper
#staticmethod
cdef WrapperClass new_struct():
"""Factory function to create WrapperClass objects with
newly allocated my_c_struct"""
cdef my_c_struct *_ptr = <my_c_struct *>malloc(sizeof(my_c_struct))
if _ptr is NULL:
raise MemoryError
_ptr.a = 0
_ptr.b = 0
return WrapperClass.from_ptr(_ptr, owner=True)
I get the following error when I try to compile:
WrapperClass.pxd:9:30: Self argument (my_c_struct *) of C method 'from_ptr' does not match parent type (WrapperClass)
Is it not possible to use static classes as a "class method" for an extension that can be cimported into another module? This sample code is a stand-in for a larger project I'm working on where I'd like to have multiple ways of constructing a cpp Cython class, some of which would use cpp data structures as arguments (which the documentation notes can only be done using factory functions).
I had this exact same issue, found this thread, and the last missing piece was in the comment by DavidW.
As for the bint owner problem, I did something that made the compiler tell me to replace the default argument with either ? or *.
So, the complete files are:
ctypedef struct my_c_struct:
int a
int b
cdef class WrapperClass:
cdef my_c_struct *_ptr
cdef bint ptr_owner
#staticmethod
cdef WrapperClass from_ptr(my_c_struct *_ptr, bint owner=*)
#staticmethod
cdef WrapperClass new_struct()
and:
# cython: language_level=3
from libc.stdlib cimport malloc, free
cdef class WrapperClass:
"""A wrapper class for a C/C++ data structure"""
def __cinit__(self):
self.ptr_owner = False
def __dealloc__(self):
# De-allocate if not null and flag is set
if self._ptr is not NULL and self.ptr_owner is True:
free(self._ptr)
self._ptr = NULL
# Extension class properties
#property
def a(self):
return self._ptr.a if self._ptr is not NULL else None
#property
def b(self):
return self._ptr.b if self._ptr is not NULL else None
#staticmethod
cdef WrapperClass from_ptr(my_c_struct *_ptr, bint owner=False):
"""Factory function to create WrapperClass objects from
given my_c_struct pointer.
Setting ``owner`` flag to ``True`` causes
the extension type to ``free`` the structure pointed to by ``_ptr``
when the wrapper object is deallocated."""
# Call to __new__ bypasses __init__ constructor
cdef WrapperClass wrapper = WrapperClass.__new__(WrapperClass)
wrapper._ptr = _ptr
wrapper.ptr_owner = owner
return wrapper
#staticmethod
cdef WrapperClass new_struct():
"""Factory function to create WrapperClass objects with
newly allocated my_c_struct"""
cdef my_c_struct *_ptr = <my_c_struct *>malloc(sizeof(my_c_struct))
if _ptr is NULL:
raise MemoryError
_ptr.a = 0
_ptr.b = 0
return WrapperClass.from_ptr(_ptr, owner=True)
Is there a way to speed it up? takes too long on realy large datasets...
"matrice" is a list of numpy arrays with no solid lenght (some could be 1-5 elements longer or shorter)
def replaceScaleBelowZero(arr):
if np.amax(arr)<=0:
arr[arr<=0] = minmax_scale(arr[arr<=0],(min_thresh*0.75,min_thresh))
elif np.min(arr)<=0:
arr[arr<=0] = minmax_scale(arr[arr<=0],(min_thresh*0.75,min_thresh))
return arr
def replaceScaleBelowMinThresh(arr):
if np.amax(arr)<min_thresh:
arr[arr<sl_thresh] = minmax_scale(arr[arr<min_thresh],(min_thresh*0.75,min_thresh))
elif np.min(arr)<min_thresh:
arr[arr<min_thresh] = minmax_scale(arr[arr<min_thresh],(min_thresh*0.75,min_thresh))
return arr
matrice = [replaceScaleBelowZero(slice_ ) for slice_ in matrice ]
matrice = [replaceScaleBelowMinThresh(slice_ ) for slice_ in matrice ]
sklearn.preprocessing.minmax_scale use a lot of checkings. If you rewrite your
arr[arr<=0] = minmax_scale(arr[arr<=0],(min_thresh*0.75,min_thresh))
as
a = arr[arra<=0]
a -= a.min()
a /= a.max()
a *= (0.25 * min_thresh)
a += 0.75 * min_thresh
(assuming arr is 1d) it should be faster. If it works I think it can be optimized further with rewriting this -=, /=, *=, += with just two such opertaions.
In your second function you use
arr[arr<sl_thresh] = minmax_scale(arr[arr<min_thresh] ...
if sl_thresh != min_thresh this can give errors. If sl_thresh = min_thresh I guess you can drop you if-else clause as your ValueError likely was raised by sklearn.
I have some functions like the below sample. The objective is to pass the first function as an argument to the second function and use numba to speed it up. But if I turn the cache=True for the second function, there will be error message:
TypeError: can't pickle weakref objects
My numba version is 0.49.0
Is there any solution/alternative solution to this problem so that I can cache the compilation?
#njit(cache=True)
def func_test(t1, t2):
return np.corrcoef(np.argsort(np.argsort(t1)), np.argsort(np.argsort(t2)))
#njit(cache=True)
def test_func_input2(func_1):
t = list(range(500))
t2 = list(range(500))
for i in range(1000):
t.pop(0)
t2.pop(0)
t.append(i)
t2.append(i)
a = np.array(t)
b = np.array(t2)
x = func_1(a, b)
return x
if __name__ == '__main__':
import numpy as np
test_func_input2(func_test)
Following is the C function para_trans_test.
void para_trans_test(char x [] [100])
{
strncpy(x[0],"zzz",100);
}
Following is the python code which doesnt work.
lib.para_trans_test.argtypes= [ctypes.POINTER(ctypes.c_char_p)]
numParams=2
L=numpy.array(['xxx','yyy'])
print(type(L))
py_s1 = (ctypes.c_char_p * numParams) ()
py_s1[:]=L
print("py_s1=",py_s1)
lib.para_trans_test(py_s1)
print(py_s1)
Initially array L is ('xxx','yyy').
After calling the C function para_trans_test I want array L to be ('zzz','yyy')
The argument type is wrong. POINTER(c_char_p) is equivalent to char**. What is needed is a pointer to a c_char array:
Test DLL:
#include <string.h>
__declspec(dllexport) void para_trans_test(char x [] [100])
{
strncpy(x[0],"zzz",100);
}
Python:
from ctypes import *
lib = CDLL('test')
lib.para_trans_test.argtypes = [POINTER(c_char * 100)]
py_s1 = (c_char * 100 * 2)()
py_s1[0].value = b'xxx'
py_s1[1].value = b'yyy'
print(py_s1[0].value,py_s1[1].value)
lib.para_trans_test(py_s1)
print(py_s1[0].value,py_s1[1].value)
Output:
b'xxx' b'yyy'
b'zzz' b'yyy'