Public channel for discussing Numba usage. Don't post confidential info here! Consider posting questions to: https://numba.discourse.group/ !
Hi, I'm trying to initialize a numba.typed.Dict()
in the @lower_builtin decorated constructor but am not able to do so because it's saying labels is not an iterable:
@lower_builtin(World, types.ListType(types.string))
def impl_world(context, builder, sig, args):
typ = sig.return_type
world = cgutils.create_struct_proxy(typ)(context, builder)
labels = args[0]
world.labels = labels
for l in numba.typed.List(labels):
world.world[l] = 0
return world._getvalue()
How can I make labels iterable and thereby initialize world.world in this constructor?
@register_jitable(_nrt=False)
, I feel I know what is register_jitable, but what is _nrt=False
? I only found a comment: disable runtime allocation
, which means (1) the decorated function used in nopython mode doesn't need memory allocation from NRT
, or (2) if the decorated function wants to allocate memory, we use another way, rather than NRT
?
environ["NUMBA_DEBUG_NRT"] = "1"
, uncomment #undef NDEBUG
, and changing to #if 1
in the line below, then recompiled but not worked, only print out other NRT info, e.g., NRT_Allocate_External
, not any meminfo refct info.void NRT_MemInfo_dump(NRT_MemInfo *mi, FILE *out) {
fprintf(out, "MemInfo %p refcount %zu\n", mi, mi->refct);
}
@numba.njit
import numba
from numba.typed import Dict as NumbaDict
def _map_1D_nb(a: np.ndarray, out: np.ndarray, d: dict) -> np.ndarray:
for i in np.arange(a.shape[0]):
out[i] = d[a[i]]
@numba.njit
def _map_2D_nb(a: np.ndarray, out: np.ndarray, d: dict) -> np.ndarray:
for i in np.arange(a.shape[0]):
for j in np.arange(a.shape[1]):
out[i, j] = d[a[i, j]]
def _map(a: np.ndarray, d: dict) -> np.ndarray:
out_value_type = numba.typeof(next(iter(d.values())))
nb_d = NumbaDict.empty(
key_type=numba.from_dtype(a.dtype),
value_type=out_value_type,
)
for k, v in d.items():
nb_d[k] = v
out = np.empty_like(a, dtype=str(out_value_type))
if a.ndim == 1:
_map_1D_nb(a, out, nb_d)
elif a.ndim == 2:
_map_2D_nb(a, out, nb_d)
else:
raise ValueError("ndim=1 or ndim=2 supported")
return out
Hi, I'm facing a small problem: I have extended numba for a custom class node
. node
has 1 attribute that is a numba TypedDict. When construct a node
object inside a jitted function and I pass a typed Dict into the constructor, the object gets instantiated correctly. But then at the end of the function, when I return node
, I get a segmentation fault
. This is because the typed Dict seems to have gone out of scope and therefore I can no longer access the node's attribute anymore. How can I prevent the typed Dict from going out of scope? Here's a sample:
@numba.njit
def f():
d = numba.typed.Dict.empty(key_type=label_type, value_type=interval_type)
n = Node(d)
return n
a = f()
I get a segmentation fault. (label_type
and interval_type
are classes that have been extended for numba as well)
@numba.njit
def f(d):
n = Node(d)
return n
d = numba.typed.Dict.empty(key_type=label_type, value_type=interval_type)
a = f(d)
def test(arr,string):
y = 0
for x in arr:
if x ==string:
c1 = y
break
y += 1
return c1 ```
should I be worried about this?
NumbaIRAssumptionWarning: variable 'bund_idx.8081' is not in scope.
This warning came from an internal pedantic check. Please report the warning
message and traceback, along with a minimal reproducer at:
Compilation finishes, and calculations seem fine at the end.
from numba import njit
@njit
def calcPi(n):
k, a, b, a1, b1 = 2, 4, 1, 12, 4
while n > 0:
p, q, k = k * k, 2 * k + 1, k + 1
a, b, a1, b1 = a1, b1, p * a + q * a1, p * b + q * b1
d, d1 = a / b, a1 / b1
while d == d1 and n > 0:
yield int(d)
n -= 1
a, a1 = 10 * (a % b), 10 * (a1 % b1)
d, d1 = a / b, a1 / b1
pi_digits = calcPi(100000)
for d in pi_digits:
print(str(d), end='', flush=True)
I would like to AOT compile some jit functions, and use the functions in another jit functions on the fly. I generated a module by AOT compilation. The exported jit function signature is <class 'builtin_function_or_method'>
rather than <class 'numba.core.registry.CPUDispatcher'>
, so I cannot use it in another jit function.
import numba
from numba.pycc import CC
cc = CC('test')
cc.verbose = True
@cc.export("test", numba.typeof(0)(numba.typeof(0)))
def test(a):
return a + 10
cc.compile()
import test
test.test(10) # works.
TEST = test.test
@numba.njit(numba.typeof(0)(numba.typeof(0)))
def test3(x):
return TEST(x)
"""
TypingError: Failed in nopython mode pipeline (step: nopython frontend)
Untyped global name 'TEST': Cannot determine Numba type of <class 'builtin_function_or_method'>
"""
test3(10)
is there a way to make this work? like import via ctypes or more straightforward way?
np.argsort(x, kind='quicksort')
gives a different result on the same x
if x
has ties on it when it's in a numba accelerated function versus outside one. I know that quicksort is unstable, but it should give the same output for the same input, right? Is quicksort implemented differently in numba than in numpy?
How can I signature the following reflected list?
lst = [np.array([[1, 2],
[3, 4]]),
np.array([[1, 2, 3],
[4, 5, 6]]),
np.array([[1, 2],
[3, 4],
[5, 6]])]
# @nb.njit(nb.types.ListType(nb.int_[:, ::1])(nb.float64[:, :, ::1]))
@nb.njit(nb.types.List(nb.int_[:, ::1], reflected=True)(nb.float64[:, :, ::1]))
def numba_(lst):
maxx = 0
maxy = 0
for x in lst:
maxx = max(x.shape[0], maxx)
maxy = max(x.shape[1], maxy)
arr = np.zeros((len(lst), maxx, maxy))
for i in range(len(lst)):
arr[i, :lst[i].shape[0], :lst[i].shape[1]] = lst[i]
return arr
which get error:
numba.core.errors.TypingError: Failed in nopython mode pipeline (step: nopython frontend)
No conversion from array(float64, 3d, C) to ListType[array(int32, 2d, C)] for '$138return_value.1', defined at None
@njit
def build_net(layer_settings):
new_net = List()
for index, layer_size in enumerate(layer_settings):
if index == 0:
new_net.append(np.zeros((layer_size)))
continue
x = np.random.randint(low=-10, high=10, size=(layer_size, 3))
y = np.random.rand(layer_size, 3)
new_layer = np.add(x, y)
new_net.append(new_layer)
return new_net
Hi guys! I am trying to enhance some linear square estimation that will use some ~26000x10 matrix multiplication. The code mainly uses numpy arrays from pandas, and some user-defined dictionaries that can be translated to lists if needed. Twoquestions:
1) How fast will numba speed up my code? At the end of each matrix value, there is a call to a C code to compute a numerical jacobian from a model. Asking in case there is something easier that I am not seeing. The numpy calls are vectorized.
2) I am running into the following error, in a call to the function that contains the main loop, even with @jit(nopython=False):
Traceback (most recent call last):
File "<string>", line 1, in <module>
NameError: global name 'length_of_iterator' is not defined
I saw numba/numba#7395, but I can't really understand the underlaying code meaning. How should I solve the error?
Hi guys!!
I'm trying to type-cast dataframe column to numba's types to inplace-update data of the dataframe column via numba's njit function.
It works well in a case of integer type as I intended that the data on dataframe is updated.
import pandas
from numba.core import types as numba_types
from numba.typed import Dict as numba_dict
df = pandas.DataFrame(data = {
'col_int': [1, 2, 3],
})
kv_int = numba_dict.empty(
key_type=numba_types.unicode_type,
value_type=numba_types.int64[:],
)
kv_int['col_int'] = df['col_int'].to_numpy(copy=False)
print(kv_int['col_int'])
print(df)
kv_int['col_int'][1] = 11
print(kv_int['col_int'])
print(df)
[1 2 3]
col_int
0 1
1 2
2 3
[ 1 11 3]
col_int
0 1
1 11
2 3
However, in a case of string type, dataframe data didn't be updated.
import pandas
from numba.core import types as numba_types
from numba.typed import Dict as numba_dict
df = pandas.DataFrame(data = {
'col_str': ['a', 'b', 'c']
})
kv_str = numba_dict.empty(
key_type=numba_types.unicode_type,
value_type=numba_types.UnicodeCharSeq(32)[:],
)
kv_str['col_str'] = df['col_str'].to_numpy(dtype='U32', copy=False)
print(kv_str['col_str'])
print(df)
kv_str['col_str'][1] = 'k'
print(kv_str['col_str'])
print(df)
['a' 'b' 'c']
col_str
0 a
1 b
2 c
['a' 'k' 'c']
col_str
0 a
1 b
2 c
Could anyone please give any idea how to do it in string case, or give any information?
@box(types.Record)
def box_record(typ, val, c):
# Note we will create a copy of the record
# This is the only safe way.
size = ir.Constant(ir.IntType(32), val.type.pointee.count)
ptr = c.builder.bitcast(val, ir.PointerType(ir.IntType(8)))
return c.pyapi.recreate_record(ptr, size, typ.dtype, c.env_manager)
in the boxing code of types.Record
, it mentions that the copy is the only safe way. Would there be some more explanation of this somewhere? why view cannot be returned?
What I did wrong heree?
from numba import njit
import numpy as np
a = np.random.normal(1, 1, 10000)
@njit
def find_quantile(a):
qx = np.quantile(a, [0.16, 0.5, 0.84])
return qx
find_quantile(a)
The error says:
TypeError: expected dtype object, got 'numpy.dtype[float64]'
Thanks a lot
Hi, is there a way to return an instance of a jitted class from a jitted function?
import numba as nb
import numpy as np
@nb.experimental.jitclass({'val': nb.float64})
class FooClass:
def __init__(self, val):
self.val = val
@nb.njit
def jitted_func(foo_class):
return foo_class(3.0)
jitted_func(FooClass)
gives me
TypingError: Failed in nopython mode pipeline (step: nopython frontend)
non-precise type pyobject
During: typing of argument at /tmp/ipykernel_2376236/1061734744.py (11)
File "../../../../../tmp/ipykernel_2376236/1061734744.py", line 11:
<source missing, REPL/exec in use?>
This error may have been caused by the following argument(s):
- argument 0: Cannot determine Numba type of <class 'numba.experimental.jitclass.base.JitClassType'>