# Copyright (C) 2022 ISIS Rutherford Appleton Laboratory UKRI
# SPDX - License - Identifier: GPL-3.0-or-later
from functools import partial
from typing import List
import numpy
from mantidimaging.core.parallel import utility as pu
shared_list: List[numpy.ndarray] = []
[docs]
def inplace3(func, i, **kwargs):
global shared_list
func(shared_list[0][i], shared_list[1][i], shared_list[2], **kwargs)
[docs]
def inplace2(func, i, **kwargs):
global shared_list
func(shared_list[0][i], shared_list[1][i], **kwargs)
[docs]
def inplace1(func, i, **kwargs):
global shared_list
func(shared_list[0][i], **kwargs)
[docs]
def return_to_self(func, i, **kwargs):
global shared_list
shared_list[0][i] = func(shared_list[0][i], **kwargs)
[docs]
def inplace_second_2d(func, i, **kwargs):
global shared_list
func(shared_list[0][i], shared_list[1], **kwargs)
[docs]
def return_to_second(func, i, **kwargs):
global shared_list
shared_list[1] = func(shared_list[0][i], **kwargs)
[docs]
def return_to_second_at_i(func, i, **kwargs):
global shared_list
shared_list[1][i] = func(shared_list[0][i], **kwargs)
[docs]
def arithmetic(func, i):
global shared_list
func(shared_list[0][i], *shared_list[1:])
[docs]
def create_partial(func, fwd_function, **kwargs):
"""
Create a partial using functools.partial, to forward the kwargs to the
parallel execution of imap.
If you seem to be getting nans, check if the correct fwd_function is set!
:param func: Function that will be executed
:param fwd_function: The function will be forwarded through function.
:param kwargs: kwargs to forward to the function func that will be executed
:return: The decorated forwarded function, ready for further execution
"""
return partial(fwd_function, func, **kwargs)
[docs]
def execute(partial_func: partial, num_operations: int, progress=None, msg: str = '', cores=None) -> None:
"""
Executes a function in parallel with shared memory between the processes.
The array must have been created using
parallel.utility.create_shared_array(shape, dtype).
If the input array IS NOT a shared array, the data will NOT BE CHANGED!
The reason for that is that the processes don't work on the data, but on a
copy.
When they process it and return the result, THE RESULT IS NOT ASSIGNED BACK
TO REPLACE THE ORIGINAL, it is discarded.
- imap_unordered gives the images back in random order
- map and map_async do not improve speed performance
- imap seems to be the best choice
Using _ in the for _ enumerate is slightly faster, because the tuple
from enumerate isn't unpacked, and thus some time is saved.
From performance tests, the chunksize doesn't seem to make much of a
difference, but having larger chunks usually led to slower performance:
Shape: (50,512,512)
1 chunk 3.06s
2 chunks 3.05s
3 chunks 3.07s
4 chunks 3.06s
5 chunks 3.16s
6 chunks 3.06s
7 chunks 3.058s
8 chunks 3.25s
9 chunks 3.45s
:param partial_func: A function constructed using create_partial
:param num_operations: The expected number of operations - should match the number of images being processed
Also used to set the number of progress steps
:param cores: number of cores that the processing will use
:param progress: Progress instance to use for progress reporting (optional)
:param msg: Message to be shown on the progress bar
:return:
"""
if not cores:
cores = pu.get_cores()
chunksize = pu.calculate_chunksize(cores)
pu.execute_impl(num_operations, partial_func, cores, chunksize, progress, msg)
global shared_list
shared_list = []