secretflow.device.device package#

Submodules#

secretflow.device.device.base module#

Classes:

Device(device_type)

MoveConfig([spu_vis, heu_dest_party, ...])

DeviceObject(device)

class secretflow.device.device.base.Device(device_type: DeviceType)[source]#

Bases: ABC

Methods:

__init__(device_type)

Abstraction device base class.

Attributes:

device_type

Get underlying device type
__init__(device_type: DeviceType)[source]#

Abstraction device base class.

Parameters

device_type (DeviceType) – underlying device type

property device_type#

Get underlying device type

class secretflow.device.device.base.MoveConfig(spu_vis: str = 'secret', heu_dest_party: str = 'auto', heu_encoder: Union[heu.phe.IntegerEncoder, heu.phe.FloatEncoder, heu.phe.BigintEncoder] = None, heu_audit_log: str = None)[source]#

Bases: object

Attributes:

spu_vis

Deivce object visibility, SPU device only.

heu_dest_party

Where the encrypted data is located

heu_encoder

Do encode before move data to heu

heu_audit_log

file path to record audit log

Methods:

__init__([spu_vis, heu_dest_party, ...])

spu_vis: str = 'secret'#

Deivce object visibility, SPU device only. Value can be: - secret: Secret sharing with protocol spdz-2k, aby3, etc. - public: Public sharing, which means data will be replicated to each node.

Type

spu_vis (str)

heu_dest_party: str = 'auto'#

Where the encrypted data is located

heu_encoder: Union[IntegerEncoder, FloatEncoder, BigintEncoder] = None#

Do encode before move data to heu

heu_audit_log: str = None#

file path to record audit log

__init__(spu_vis: str = 'secret', heu_dest_party: str = 'auto', heu_encoder: Optional[Union[IntegerEncoder, FloatEncoder, BigintEncoder]] = None, heu_audit_log: Optional[str] = None) → None#
class secretflow.device.device.base.DeviceObject(device: Device)[source]#

Bases: ABC

Methods:

__init__(device)

Abstraction device object base class.

to(device[, config])

Device object conversion.

Attributes:

device_type

Get underlying device type
__init__(device: Device)[source]#

Abstraction device object base class.

Parameters

device (Device) – Device where this object is located.

property device_type#

Get underlying device type

to(device: Device, config: Optional[MoveConfig] = None)[source]#

Device object conversion.

Parameters

  • device (Device) – Target device

  • config – configuration of this data movement

Returns

Target device object.

Return type

DeviceObject

secretflow.device.device.heu module#

Classes:

HEUActor(heu_id, party, hekit, ...)

HEU(config, spu_field_type)

Homomorphic encryption device
class secretflow.device.device.heu.HEUActor(heu_id, party: str, hekit: Union[HeKit, DestinationHeKit], cleartext_type: dtype, encoder)[source]#

Bases: object

Methods:

__init__(heu_id, party, hekit, ...)

Init heu actor class

getitem(data, item, *[, _ray_trace_ctx])

Delegate of hnp ndarray.__getitem___()

setitem(data, key, value, *[, _ray_trace_ctx])

Delegate of hnp ndarray.__setitem___()

sum(data, *[, _ray_trace_ctx])

sum of data elements

encode(data[, edr, _ray_trace_ctx])

encode cleartext to plaintext

decode(data[, edr, _ray_trace_ctx])

decode plaintext to cleartext

encrypt(data[, heu_audit_log, _ray_trace_ctx])

Encrypt data

do_binary_op(fn_name, data1, data2, *[, ...])

perform math operation :param fn: hnp.Evaluator functions, such as hnp.Evaluator.add, hnp.Evaluator.sub
__init__(heu_id, party: str, hekit: Union[HeKit, DestinationHeKit], cleartext_type: dtype, encoder)[source]#

Init heu actor class

Parameters

  • heu_id – Heu instance id, globally unique

  • party – The party id

  • hekit – hnp.HeKit for sk_keeper or hnp.DestinationHeKit for evaluator

  • encoder –

    Encode cleartext (float value) to plaintext (big int value). available encoders:

    • phe.IntegerEncoder

    • phe.FloatEncoder

    • phe.BigintEncoder

    • phe.BatchEncoder

getitem(data, item, *, _ray_trace_ctx=None)[source]#

Delegate of hnp ndarray.__getitem___()

setitem(data, key, value, *, _ray_trace_ctx=None)[source]#

Delegate of hnp ndarray.__setitem___()

sum(data, *, _ray_trace_ctx=None)[source]#

sum of data elements

encode(data: ndarray, edr=None, *, _ray_trace_ctx=None)[source]#

encode cleartext to plaintext

Parameters

  • data – cleartext

  • edr – encoder

decode(data: PlaintextArray, edr=None, *, _ray_trace_ctx=None)[source]#

decode plaintext to cleartext

Parameters

  • data – plaintext

  • edr – encoder

encrypt(data: PlaintextArray, heu_audit_log: Optional[str] = None, *, _ray_trace_ctx=None) → CiphertextArray[source]#

Encrypt data

If the data has already been encoded, the data will be encrypted directly, you don’t have to worry about the data being encoded repeatedly

Even if the data has been encrypted, you still need to pass in the encoder param, because decryption will use it

Parameters

  • data – The data to be encrypted

  • heu_audit_log – file path to log audit info

Returns

The encrypted ndarray data

do_binary_op(fn_name, data1, data2, *, _ray_trace_ctx=None)[source]#

perform math operation :param fn: hnp.Evaluator functions, such as hnp.Evaluator.add, hnp.Evaluator.sub

Returns

numpy ndarray of HeCiphertext

class secretflow.device.device.heu.HEU(config: dict, spu_field_type)[source]#

Bases: Device

Homomorphic encryption device

Methods:

__init__(config, spu_field_type)

Initialize HEU

init()

sk_keeper_name()

evaluator_names()

get_participant(party)

Get ray actor by name

has_party(party)

__init__(config: dict, spu_field_type)[source]#

Initialize HEU

Parameters

  • config –

    HEU init config, for example

    {
    'sk_keeper': {
        'party': 'alice'
    },
    'evaluators': [{
        'party': 'bob'
    }],
    # The HEU working mode, choose from PHEU / LHEU / FHEU_ROUGH / FHEU
    'mode': 'PHEU',
    # TODO: cleartext_type should be migrated to HeObject.
    'encoding': {
        # DT_I1, DT_I8, DT_I16, DT_I32, DT_I64 or DT_FXP (default)
        'cleartext_type': "DT_FXP"
        # see https://heu.readthedocs.io/en/latest/getting_started/quick_start.html#id3 for detail
        # available encoders:
        #     - IntegerEncoder: Plaintext = Cleartext * scale
        #     - FloatEncoder (default): Plaintext = Cleartext * scale
        #     - BigintEncoder: Plaintext = Cleartext
        #     - BatchEncoder: Plaintext = Pack[Cleartext, Cleartext]
        'encoder': 'FloatEncoder'
    }
    'he_parameters': {
        'schema': 'paillier',
        'key_pair': {
            'generate': {
                'bit_size': 2048,
            },
        }
    }
}

  • spu_field_type – Field type in spu, Device.to operation requires the data scale of HEU to be aligned with SPU

init()[source]#
sk_keeper_name()[source]#
evaluator_names()[source]#
get_participant(party: str)[source]#

Get ray actor by name

has_party(party: str)[source]#

secretflow.device.device.heu_object module#

Classes:

HEUObject(device, data, location_party[, ...])

HEU Object
class secretflow.device.device.heu_object.HEUObject(device, data: ObjectRef, location_party: str, is_plain: bool = False)[source]#

Bases: DeviceObject

HEU Object

data#

The data hold by this Heu object

location#

The party where the data actually resides

is_plain#

Is the data encrypted or not

Methods:

__init__(device, data, location_party[, ...])

Abstraction device object base class.

encrypt([heu_audit_log])

Force encrypt if data is plaintext

sum()

Sum of HeObject elements over a given axis.

dump(path)

Dump ciphertext into files.
__init__(device, data: ObjectRef, location_party: str, is_plain: bool = False)[source]#

Abstraction device object base class.

Parameters

device (Device) – Device where this object is located.

encrypt(heu_audit_log: Optional[str] = None)[source]#

Force encrypt if data is plaintext

sum()[source]#

Sum of HeObject elements over a given axis.

Returns

sum_along_axis

dump(path)[source]#

Dump ciphertext into files.

secretflow.device.device.pyu module#

Classes:

PYUObject(device, data)

PYU device object.

PYU(party[, node])

PYU is the device doing computation in single domain.
class secretflow.device.device.pyu.PYUObject(device: PYU, data: ObjectRef)[source]#

Bases: DeviceObject

PYU device object.

data#

Reference to underlying data.

Type

ray.ObjectRef

Methods:

__init__(device, data)

Abstraction device object base class.
__init__(device: PYU, data: ObjectRef)[source]#

Abstraction device object base class.

Parameters

device (Device) – Device where this object is located.

class secretflow.device.device.pyu.PYU(party: str, node: str = '')[source]#

Bases: Device

PYU is the device doing computation in single domain.

Essentially PYU is a python worker who can execute any python code.

Methods:

__init__(party[, node])

PYU contructor.
__init__(party: str, node: str = '')[source]#

PYU contructor.

Parameters

  • party (str) – Party name where this device is located.

  • node (str, optional) – Node name where the device is located. Defaults to “”.

secretflow.device.device.register module#

Classes:

DeviceType(value)

An enumeration.

Registrar()

Device kernel registry

Functions:

register(device_type[, op_name])

Register device kernel

dispatch(name, self, *args, **kwargs)

Dispatch device kernel.
class secretflow.device.device.register.DeviceType(value)[source]#

Bases: IntEnum

An enumeration.

Attributes:

PYU

SPU

TEE

HEU

NUM

PYU = 0#
SPU = 1#
TEE = 2#
HEU = 3#
NUM = 4#
class secretflow.device.device.register.Registrar[source]#

Bases: object

Device kernel registry

Methods:

__init__()

register(device_type, name, op)

Register device kernel.

dispatch(device_type, name, *args, **kwargs)

Dispatch device kernel.
__init__() → None[source]#
register(device_type: DeviceType, name: str, op: Callable)[source]#

Register device kernel.

Parameters

  • device_type (DeviceType) – Device type.

  • name (str) – Op kernel name.

  • op (Callable) – Op kernel implementaion.

Raises

KeyError – Duplicate device kernel registered.

dispatch(device_type: DeviceType, name: str, *args, **kwargs)[source]#

Dispatch device kernel.

Parameters

  • device_type (DeviceType) – Device type.

  • name (str) – Op kernel name.

Raises

KeyError – Device Kernel not registered.

Returns

Kernel execution result.

secretflow.device.device.register.register(device_type: DeviceType, op_name: Optional[str] = None)[source]#

Register device kernel

Parameters

  • device_type (DeviceType) – Device type.

  • op_name (str, optional) – Op kernel name. Defaults to None.

secretflow.device.device.register.dispatch(name: str, self, *args, **kwargs)[source]#

Dispatch device kernel.

Parameters

  • name (str) – Kernel name.

  • self (Device) – Traget deivce.

Returns

Kernel execution result.

secretflow.device.device.spu module#

Classes:

SPUValueMeta(shape, dtype, vtype)

The metadata of a SPU value, which is a Numpy array or equivalent.

SPUObject(device, meta, shares)

SPUIO(runtime_config, world_size)

SPUCompilerNumReturnsPolicy(value)

Tell SPU device how to decide num of returns of called function.

SPU(cluster_def[, link_desc, name])

class secretflow.device.device.spu.SPUValueMeta(shape: ~typing.Sequence[int], dtype: ~numpy.dtype, vtype: <google.protobuf.internal.enum_type_wrapper.EnumTypeWrapper object at 0x7fbaa6dd2b80>)[source]#

Bases: object

The metadata of a SPU value, which is a Numpy array or equivalent.

Attributes:

shape

dtype

vtype

Methods:

__init__(shape, dtype, vtype)

shape: Sequence[int]#
dtype: dtype#
vtype: <google.protobuf.internal.enum_type_wrapper.EnumTypeWrapper object at 0x7fbaa6dd2b80>#
__init__(shape: ~typing.Sequence[int], dtype: ~numpy.dtype, vtype: <google.protobuf.internal.enum_type_wrapper.EnumTypeWrapper object at 0x7fbaa6dd2b80>) → None#
class secretflow.device.device.spu.SPUObject(device: Device, meta: ObjectRef, shares: Sequence[ObjectRef])[source]#

Bases: DeviceObject

Methods:

__init__(device, meta, shares)

SPUObject refers to a Python Object which could be flattened to a list of SPU Values.
__init__(device: Device, meta: ObjectRef, shares: Sequence[ObjectRef])[source]#

SPUObject refers to a Python Object which could be flattened to a list of SPU Values. A SPU value is a Numpy array or equivalent. e.g.

1. If referred Python object is [1,2,3] Then meta would be referred to a single SPUValueMeta, and shares is a list of referrence to pieces of share of [1,2,3].

2. If referred Python object is {‘a’: 1, ‘b’: [3, np.array(…)]} The meta would be referred to something like {‘a’: SPUValueMeta1, ‘b’: [SPUValueMeta2, SPUValueMeta3]} Each element of shares would be referred to something like {‘a’: share1, ‘b’: [share2, share3]}

3. shares is a list of ObjectRef to share slices while these share slices are not necessarily located at SPU device. The data transfer would only happen when SPU device consumes SPU objects.

Parameters

  • meta – Ref to the metadata.

  • refs (Sequence[ray.ObjectRef]) – Refs to shares of data.

class secretflow.device.device.spu.SPUIO(runtime_config: RuntimeConfig, world_size: int)[source]#

Bases: object

Methods:

__init__(runtime_config, world_size)

A wrapper of spu.Io.

make_shares(data, vtype)

Convert a Python object to meta and shares of a SPUObject.

reconstruct(shares)

Convert shares of a SPUObject to the origin Python object.
__init__(runtime_config: RuntimeConfig, world_size: int) → None[source]#

A wrapper of spu.Io.

Parameters

  • runtime_config (RuntimeConfig) – runtime_config of SPU device.

  • world_size (int) – world_size of SPU device.

make_shares(data: ~typing.Any, vtype: <google.protobuf.internal.enum_type_wrapper.EnumTypeWrapper object at 0x7fbaa6dd2b80>) → Tuple[Any, List[Any]][source]#

Convert a Python object to meta and shares of a SPUObject.

Parameters

  • data (Any) – Any Python object.

  • vtype (Visibility) – Visibility

Returns

meta and shares of a SPUObject

Return type

Tuple[Any, List[Any]]

reconstruct(shares: List[Any]) → Any[source]#

Convert shares of a SPUObject to the origin Python object.

Parameters

shares (List[Any]) – Shares

Returns

the origin Python object.

Return type

Any

class secretflow.device.device.spu.SPUCompilerNumReturnsPolicy(value)[source]#

Bases: Enum

Tell SPU device how to decide num of returns of called function.

Attributes:

FROM_COMPILER

num of returns is from compiler result.

FROM_USER

If users are sure that returns is a list, they could specify the length of list.

SINGLE

num of returns is fixed to 1.
FROM_COMPILER = 'from_compiler'#

num of returns is from compiler result.

FROM_USER = 'from_user'#

If users are sure that returns is a list, they could specify the length of list.

SINGLE = 'single'#

num of returns is fixed to 1.

class secretflow.device.device.spu.SPU(cluster_def: Dict, link_desc: Optional[Dict] = None, name: str = 'SPU')[source]#

Bases: Device

Methods:

__init__(cluster_def[, link_desc, name])

SPU device constructor.

init()

Init SPU runtime in each party

reset()

Reset spu to clear corrupted internal state, for test only

psi_df(key, dfs, receiver[, protocol, ...])

Private set intersection with DataFrame.

psi_csv(key, input_path, output_path, receiver)

Private set intersection with csv file.

psi_join_df(key, dfs, receiver, join_party)

Private set intersection with csv file.

psi_join_csv(key, input_path, output_path, ...)

Private set intersection with csv file.
__init__(cluster_def: Dict, link_desc: Optional[Dict] = None, name: str = 'SPU')[source]#

SPU device constructor.

Parameters

  • cluster_def –

    SPU cluster definition. More details refer to SPU runtime config.

    For example

    {
    'nodes': [
        {
            'party': 'alice',
            'id': 'local:0',
            # The address for other peers.
            'address': '127.0.0.1:9001',
            # The listen address of this node.
            # Optional. Address will be used if listen_address is empty.
            'listen_address': ''
        },
        {
            'party': 'bob',
            'id': 'local:1',
            'address': '127.0.0.1:9002',
            'listen_address': ''
        },
    ],
    'runtime_config': {
        'protocol': spu.spu_pb2.SEMI2K,
        'field': spu.spu_pb2.FM128,
        'sigmoid_mode': spu.spu_pb2.RuntimeConfig.SIGMOID_REAL,
    }
}

  • link_desc –

    optional. A dict specifies the link parameters. Available parameters are:

    1. connect_retry_times

    2. connect_retry_interval_ms

    3. recv_timeout_ms

    4. http_max_payload_size

    5. http_timeout_ms

    6. throttle_window_size

    7. brpc_channel_protocol refer to https://github.com/apache/incubator-brpc/blob/master/docs/en/client.md#protocols

    8. brpc_channel_connection_type refer to https://github.com/apache/incubator-brpc/blob/master/docs/en/client.md#connection-type

init()[source]#

Init SPU runtime in each party

reset()[source]#

Reset spu to clear corrupted internal state, for test only

psi_df(key: Union[str, List[str], Dict[Device, List[str]]], dfs: List[PYUObject], receiver: str, protocol='KKRT_PSI_2PC', precheck_input=True, sort=True, broadcast_result=True, bucket_size=1048576, curve_type='CURVE_25519')[source]#

Private set intersection with DataFrame.

Parameters

  • key (str, List[str], Dict[Device, List[str]]) – Column(s) used to join.

  • dfs (List[PYUObject]) – DataFrames to be joined, which

  • runtimes. (should be colocated with SPU) –

  • receiver (str) – Which party can get joined data, others will get None.

  • protocol (str) – PSI protocol.

  • precheck_input (bool) – Whether to check input data before join.

  • sort (bool) – Whether sort data by key after join.

  • broadcast_result (bool) – Whether to broadcast joined data to all parties.

  • bucket_size (int) – Specified the hash bucket size used in psi.

  • memory. (Larger values consume more) –

  • curve_type (str) – curve for ecdh psi

Returns

Joined DataFrames with order reserved.

Return type

List[PYUObject]

psi_csv(key: Union[str, List[str], Dict[Device, List[str]]], input_path: Union[str, Dict[Device, str]], output_path: Union[str, Dict[Device, str]], receiver: str, protocol='KKRT_PSI_2PC', precheck_input=True, sort=True, broadcast_result=True, bucket_size=1048576, curve_type='CURVE_25519')[source]#

Private set intersection with csv file.

Parameters

  • key (str, List[str], Dict[Device, List[str]]) – Column(s) used to join.

  • input_path – CSV files to be joined, comma seperated and contains header.

  • output_path – Joined csv files, comma seperated and contains header.

  • receiver (str) – Which party can get joined data.

  • -1. (Others won't generate output file and intersection_count get) –

  • protocol (str) – PSI protocol.

  • precheck_input (bool) – Whether check input data before joining,

  • now (for) –

  • duplicate. (it will check if key) –

  • sort (bool) – Whether sort data by key after joining.

  • broadcast_result (bool) – Whether broadcast joined data to all parties.

  • bucket_size (int) – Specified the hash bucket size used in psi.

  • memory. (Larger values consume more) –

Returns

PSI reports output by SPU with order reserved.

Return type

List[Dict]

psi_join_df(key: Union[str, List[str], Dict[Device, List[str]]], dfs: List[PYUObject], receiver: str, join_party: str, protocol='KKRT_PSI_2PC', precheck_input=True, bucket_size=1048576, curve_type='CURVE_25519')[source]#

Private set intersection with csv file.

Parameters

  • key (str, List[str], Dict[Device, List[str]]) – Column(s) used to join.

  • dfs (List[PYUObject]) – DataFrames to be joined, which should be colocated with SPU runtimes.

  • receiver (str) – Which party can get joined data. Others won’t generate output file and intersection_count get -1

  • join_party (str) – party can get joined data

  • protocol (str) – PSI protocol.

  • precheck_input (bool) – Whether check input data before joining, for now, it will check if key duplicate.

  • bucket_size (int) – Specified the hash bucket size used in psi. Larger values consume more memory.

  • curve_type (str) – curve for ecdh psi

Returns

Joined DataFrames with order reserved.

Return type

List[PYUObject]

psi_join_csv(key: Union[str, List[str], Dict[Device, List[str]]], input_path: Union[str, Dict[Device, str]], output_path: Union[str, Dict[Device, str]], receiver: str, join_party: str, protocol='KKRT_PSI_2PC', precheck_input=True, bucket_size=1048576, curve_type='CURVE_25519')[source]#

Private set intersection with csv file.

Parameters

  • key (str, List[str], Dict[Device, List[str]]) – Column(s) used to join.

  • input_path – CSV files to be joined, comma seperated and contains header.

  • output_path – Joined csv files, comma seperated and contains header.

  • receiver (str) – Which party can get joined data. Others won’t generate output file and intersection_count get -1

  • join_party (str) – party can get joined data

  • protocol (str) – PSI protocol.

  • precheck_input (bool) – Whether check input data before joining, for now, it will check if key duplicate.

  • bucket_size (int) – Specified the hash bucket size used in psi. Larger values consume more memory.

  • curve_type (str) – curve for ecdh psi

Returns

PSI reports output by SPU with order reserved.

Return type

List[Dict]

secretflow.device.device.type_traits module#

Functions:

spu_fxp_precision(field_type)

Fixed point integer default precision bits

spu_fxp_size(field_type)

Fixed point integer size in bytes

heu_datatype_to_spu(heu_dt)

spu_datatype_to_heu(spu_dt)

heu_datatype_to_numpy(heu_dt)

secretflow.device.device.type_traits.spu_fxp_precision(field_type)[source]#

Fixed point integer default precision bits

secretflow.device.device.type_traits.spu_fxp_size(field_type)[source]#

Fixed point integer size in bytes

secretflow.device.device.type_traits.heu_datatype_to_spu(heu_dt)[source]#
secretflow.device.device.type_traits.spu_datatype_to_heu(spu_dt)[source]#
secretflow.device.device.type_traits.heu_datatype_to_numpy(heu_dt) → dtype[source]#

Module contents#