mirror of
https://github.com/luxfi/fhe.git
synced 2026-07-26 23:16:08 +00:00
Complete Lux FHE rebrand
This commit is contained in:
Zach Kelling
@@ -431,7 +431,7 @@ function reveal(bytes32 requestId) external view returns (bytes memory result, b
|
||||
- rng-game, rps-game, secret-santa, voting
|
||||
|
||||
**⚠️ Not using @luxfi/contracts:**
|
||||
- `dapps/` - Uses `@fhevm/solidity` (Zama's library)
|
||||
- `dapps/` - Uses `@fhevm/solidity` (lux's library)
|
||||
- `poker/`, `kuhn-poker/` - Custom implementation
|
||||
- `ticketing/`, `tickets/` - Package manager issues
|
||||
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
# Lux FHE
|
||||
|
||||
[](https://pkg.go.dev/github.com/luxfi/fhe)
|
||||
[](https://github.com/luxfi/fhe/actions/workflows/ci.yml)
|
||||
|
||||
Fully Homomorphic Encryption (FHE) library for the Lux Network.
|
||||
|
||||
## Features
|
||||
|
||||
- **TFHE** - Fast Boolean operations on encrypted data
|
||||
- **CKKS** - Approximate arithmetic on encrypted vectors
|
||||
- **GPU Acceleration** - Metal and CUDA support via luxfi/gpu
|
||||
- **Lazy Carry Propagation** - Efficient integer arithmetic
|
||||
- **Multi-Party Computation** - Threshold decryption support
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
go get github.com/luxfi/fhe
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### TFHE Boolean Gates
|
||||
|
||||
```go
|
||||
import "github.com/luxfi/fhe"
|
||||
|
||||
// Generate keys
|
||||
params := fhe.DefaultTFHEParams()
|
||||
keys, _ := fhe.GenerateTFHEKeys(params)
|
||||
|
||||
// Encrypt bits
|
||||
ct1, _ := fhe.TFHEEncrypt(keys, true)
|
||||
ct2, _ := fhe.TFHEEncrypt(keys, false)
|
||||
|
||||
// Homomorphic operations
|
||||
ctAnd, _ := fhe.TFHEAnd(keys, ct1, ct2)
|
||||
ctOr, _ := fhe.TFHEOr(keys, ct1, ct2)
|
||||
ctNot, _ := fhe.TFHENot(keys, ct1)
|
||||
|
||||
// Decrypt
|
||||
result, _ := fhe.TFHEDecrypt(keys, ctAnd) // false
|
||||
```
|
||||
|
||||
### Integer Operations
|
||||
|
||||
```go
|
||||
import "github.com/luxfi/fhe"
|
||||
|
||||
// Encrypt integers
|
||||
a, _ := fhe.EncryptInt(keys, 42, 8) // 8-bit integer
|
||||
b, _ := fhe.EncryptInt(keys, 10, 8)
|
||||
|
||||
// Arithmetic
|
||||
sum, _ := fhe.IntAdd(keys, a, b)
|
||||
diff, _ := fhe.IntSub(keys, a, b)
|
||||
prod, _ := fhe.IntMul(keys, a, b)
|
||||
|
||||
// Comparison
|
||||
isGreater, _ := fhe.IntGreaterThan(keys, a, b)
|
||||
```
|
||||
|
||||
### CKKS Approximate Arithmetic
|
||||
|
||||
```go
|
||||
import "github.com/luxfi/fhe"
|
||||
|
||||
// Setup CKKS
|
||||
params := fhe.DefaultCKKSParams()
|
||||
keys, _ := fhe.GenerateCKKSKeys(params)
|
||||
|
||||
// Encrypt vectors
|
||||
values := []float64{1.0, 2.0, 3.0, 4.0}
|
||||
ct, _ := fhe.CKKSEncrypt(keys, values)
|
||||
|
||||
// Operations
|
||||
scaled, _ := fhe.CKKSMulScalar(ct, 2.5)
|
||||
rotated, _ := fhe.CKKSRotate(keys, ct, 1)
|
||||
|
||||
// Decrypt
|
||||
result, _ := fhe.CKKSDecrypt(keys, scaled)
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### Key Generation
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `GenerateTFHEKeys(params)` | Generate TFHE key set |
|
||||
| `GenerateCKKSKeys(params)` | Generate CKKS key set |
|
||||
|
||||
### TFHE Operations
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `TFHEEncrypt(keys, bit)` | Encrypt a bit |
|
||||
| `TFHEDecrypt(keys, ct)` | Decrypt a ciphertext |
|
||||
| `TFHEAnd(keys, a, b)` | Homomorphic AND |
|
||||
| `TFHEOr(keys, a, b)` | Homomorphic OR |
|
||||
| `TFHENot(keys, a)` | Homomorphic NOT |
|
||||
| `TFHENand(keys, a, b)` | Homomorphic NAND |
|
||||
| `TFHEXor(keys, a, b)` | Homomorphic XOR |
|
||||
| `TFHEBootstrap(keys, ct)` | Refresh ciphertext noise |
|
||||
|
||||
### Integer Operations
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `EncryptInt(keys, value, bits)` | Encrypt integer |
|
||||
| `DecryptInt(keys, ct)` | Decrypt integer |
|
||||
| `IntAdd(keys, a, b)` | Addition |
|
||||
| `IntSub(keys, a, b)` | Subtraction |
|
||||
| `IntMul(keys, a, b)` | Multiplication |
|
||||
| `IntGreaterThan(keys, a, b)` | Comparison |
|
||||
|
||||
### CKKS Operations
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `CKKSEncrypt(keys, values)` | Encrypt vector |
|
||||
| `CKKSDecrypt(keys, ct)` | Decrypt vector |
|
||||
| `CKKSAdd(a, b)` | Vector addition |
|
||||
| `CKKSMul(keys, a, b)` | Vector multiplication |
|
||||
| `CKKSRotate(keys, ct, steps)` | Rotate slots |
|
||||
|
||||
## GPU Acceleration
|
||||
|
||||
Enable GPU acceleration via the luxfi/gpu package:
|
||||
|
||||
```go
|
||||
import (
|
||||
"github.com/luxfi/fhe"
|
||||
"github.com/luxfi/gpu"
|
||||
)
|
||||
|
||||
// Initialize GPU session
|
||||
sess, _ := gpu.DefaultSession()
|
||||
|
||||
// Use GPU-accelerated FHE
|
||||
fheOps := sess.FHE()
|
||||
result, _ := fheOps.NTTForward(poly, modulus)
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
See the `examples/` directory for complete examples:
|
||||
|
||||
- `examples/tfhe_basic/` - Basic TFHE operations
|
||||
- `examples/integer_ops/` - Integer arithmetic
|
||||
- `examples/ckks_ml/` - Machine learning on encrypted data
|
||||
- `examples/multiparty/` - Threshold decryption
|
||||
|
||||
## Documentation
|
||||
|
||||
- [API Reference](https://pkg.go.dev/github.com/luxfi/fhe)
|
||||
- [Workshop Materials](docs/workshop/)
|
||||
- [Resources](docs/resources/)
|
||||
|
||||
## Testing
|
||||
|
||||
```bash
|
||||
# Run all tests
|
||||
go test ./...
|
||||
|
||||
# With GPU
|
||||
CGO_ENABLED=1 go test ./...
|
||||
|
||||
# Benchmarks
|
||||
go test -bench=. ./...
|
||||
```
|
||||
|
||||
## Performance
|
||||
|
||||
Performance varies by operation and key parameters:
|
||||
|
||||
| Operation | Time (128-bit security) |
|
||||
|-----------|-------------------------|
|
||||
| TFHE Gate | ~10ms |
|
||||
| Bootstrap | ~50ms |
|
||||
| CKKS Add | <1ms |
|
||||
| CKKS Mul | ~5ms |
|
||||
| NTT (N=2^16) | ~1ms (GPU) |
|
||||
|
||||
## License
|
||||
|
||||
BSD 3-Clause. See [LICENSE](LICENSE).
|
||||
|
||||
## Links
|
||||
|
||||
- [Lux FHE Documentation](https://docs.luxfhe.ai)
|
||||
- [Research Papers](docs/resources/README.md)
|
||||
- [GitHub Issues](https://github.com/luxfi/fhe/issues)
|
||||
+6
-9
@@ -473,6 +473,7 @@ func (eval *BitwiseEvaluator) Eq(a, b *BitCiphertext) (*Ciphertext, error) {
|
||||
}
|
||||
|
||||
// Lt returns encrypted 1 if a < b, 0 otherwise (unsigned)
|
||||
// Optimized using CMPCOMBINE gate to reduce bootstraps from ~37 to ~23 for 8-bit
|
||||
func (eval *BitwiseEvaluator) Lt(a, b *BitCiphertext) (*Ciphertext, error) {
|
||||
if a.numBits != b.numBits {
|
||||
return nil, fmt.Errorf("bit count mismatch")
|
||||
@@ -487,9 +488,8 @@ func (eval *BitwiseEvaluator) Lt(a, b *BitCiphertext) (*Ciphertext, error) {
|
||||
|
||||
// Start from MSB
|
||||
for i := numBits - 1; i >= 0; i-- {
|
||||
// a[i] < b[i]: NOT(a[i]) AND b[i]
|
||||
notA := eval.eval.NOT(a.bits[i])
|
||||
bitLt, err := eval.eval.AND(notA, b.bits[i])
|
||||
// a[i] < b[i]: NOT(a[i]) AND b[i] = ANDNY(a[i], b[i])
|
||||
bitLt, err := eval.eval.ANDNY(a.bits[i], b.bits[i])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -504,12 +504,9 @@ func (eval *BitwiseEvaluator) Lt(a, b *BitCiphertext) (*Ciphertext, error) {
|
||||
isLess = bitLt
|
||||
isEqual = bitEq
|
||||
} else {
|
||||
// isLess = isLess OR (isEqual AND bitLt)
|
||||
eqAndLt, err := eval.eval.AND(isEqual, bitLt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
isLess, err = eval.eval.OR(isLess, eqAndLt)
|
||||
// Use CMPCOMBINE to compute: isLess OR (isEqual AND bitLt)
|
||||
// This replaces 2 bootstraps (AND + OR) with 1 bootstrap
|
||||
isLess, err = eval.eval.CMPCOMBINE(isLess, isEqual, bitLt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ Install LuxFHE Devnet in MetaMask:
|
||||
- Network name: LuxFHE Devnet
|
||||
- RPC URL: https://devnet.luxfhe.ai
|
||||
- Chain ID: 8009
|
||||
- Currency symbol: ZAMA
|
||||
- Currency symbol: LUX
|
||||
- Block explorer: https://main.explorer.luxfhe.ai/
|
||||
|
||||
## Encrypted ERC20
|
||||
|
||||
@@ -23,13 +23,13 @@ class FHEVM:
|
||||
print("Fetching global public key")
|
||||
self.network_encryption_key = self.w3.eth.call({"to": PUBLIC_KEY_CONTRACT})
|
||||
|
||||
with open("/tmp/zama_network_public_key", mode="wb") as f:
|
||||
with open("/tmp/lux_network_public_key", mode="wb") as f:
|
||||
f.write(self.network_encryption_key)
|
||||
res = os.system(
|
||||
f"fhevm-tfhe-cli public-encrypt-integer32 -v {plaintext} -c /tmp/zama_ciphertext -p /tmp/zama_network_public_key"
|
||||
f"fhevm-tfhe-cli public-encrypt-integer32 -v {plaintext} -c /tmp/lux_ciphertext -p /tmp/lux_network_public_key"
|
||||
)
|
||||
assert res == 0, "fail to execute fhe-tool"
|
||||
with open("/tmp/zama_ciphertext", mode="rb") as file:
|
||||
with open("/tmp/lux_ciphertext", mode="rb") as file:
|
||||
ciphertext = file.read()
|
||||
assert len(ciphertext) == 8404
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ from web3.middleware import construct_sign_and_send_raw_middleware
|
||||
|
||||
SIGNATURE_KEY_VAR = "WORKSHOP_PRIVATE_KEY"
|
||||
DEVNET_VAR = "ETHCC23_DEVNET"
|
||||
DEFAULT_DEVNET = "https://devnet.zama.ai"
|
||||
DEFAULT_DEVNET = "https://devnet.lux.network"
|
||||
|
||||
|
||||
def setup_w3():
|
||||
|
||||
@@ -238,6 +238,17 @@ func (eval *Evaluator) XNOR(ct1, ct2 *Ciphertext) (*Ciphertext, error) {
|
||||
return eval.bootstrap(doubled, eval.bsk.TestPolyXNOR)
|
||||
}
|
||||
|
||||
// CMPCOMBINE computes: isLess OR (isEqual AND bitLt) in one bootstrap
|
||||
// This is an optimized gate for comparison propagation.
|
||||
// Uses weighted encoding: 2*isLess + isEqual + bitLt, threshold at 0
|
||||
func (eval *Evaluator) CMPCOMBINE(isLess, isEqual, bitLt *Ciphertext) (*Ciphertext, error) {
|
||||
// Compute weighted sum: 2*isLess + isEqual + bitLt
|
||||
doubledIsLess := eval.doubleCiphertext(isLess)
|
||||
sum := eval.addCiphertexts(doubledIsLess, isEqual)
|
||||
sum = eval.addCiphertexts(sum, bitLt)
|
||||
return eval.bootstrap(sum, eval.bsk.TestPolyCMPCOMBINE)
|
||||
}
|
||||
|
||||
// ANDNY computes AND with negated first input: AND(NOT(a), b)
|
||||
func (eval *Evaluator) ANDNY(ct1, ct2 *Ciphertext) (*Ciphertext, error) {
|
||||
return eval.AND(eval.NOT(ct1), ct2)
|
||||
|
||||
@@ -180,6 +180,10 @@ type BootstrapKey struct {
|
||||
TestPolyID *ring.Poly
|
||||
// TestPolyMAJORITY is the test polynomial for majority vote (2 of 3)
|
||||
TestPolyMAJORITY *ring.Poly
|
||||
// TestPolyCMPCOMBINE is the test polynomial for comparison combine:
|
||||
// output = isLess OR (isEqual AND bitLt)
|
||||
// Uses weighted sum: 2*isLess + isEqual + bitLt >= 0
|
||||
TestPolyCMPCOMBINE *ring.Poly
|
||||
// Parameters
|
||||
params Parameters
|
||||
}
|
||||
@@ -354,22 +358,42 @@ func (kg *KeyGenerator) GenBootstrapKey(sk *SecretKey) *BootstrapKey {
|
||||
return -1.0
|
||||
}, scale, kg.ringQBR, -1, 1)
|
||||
|
||||
// CMPCOMBINE: computes isLess OR (isEqual AND bitLt) in one bootstrap
|
||||
// Uses weighted encoding: 2*isLess + isEqual + bitLt
|
||||
// With Q/8 encoding, weighted sum ranges:
|
||||
// - (F,F,F): 2*(-1/8) + (-1/8) + (-1/8) = -4/8 = -0.5 → FALSE
|
||||
// - (F,F,T): 2*(-1/8) + (-1/8) + (+1/8) = -2/8 = -0.25 → FALSE
|
||||
// - (F,T,F): 2*(-1/8) + (+1/8) + (-1/8) = -2/8 = -0.25 → FALSE
|
||||
// - (F,T,T): 2*(-1/8) + (+1/8) + (+1/8) = 0 → TRUE
|
||||
// - (T,F,F): 2*(+1/8) + (-1/8) + (-1/8) = 0 → TRUE
|
||||
// - (T,F,T): 2*(+1/8) + (-1/8) + (+1/8) = +2/8 = +0.25 → TRUE
|
||||
// - (T,T,F): 2*(+1/8) + (+1/8) + (-1/8) = +2/8 = +0.25 → TRUE
|
||||
// - (T,T,T): 2*(+1/8) + (+1/8) + (+1/8) = +4/8 = +0.5 → TRUE
|
||||
// Threshold at -0.125 (midpoint between -0.25 and 0) for maximum noise margin
|
||||
testPolyCMPCOMBINE := blindrot.InitTestPolynomial(func(x float64) float64 {
|
||||
if x > -0.125 {
|
||||
return 1.0
|
||||
}
|
||||
return -1.0
|
||||
}, scale, kg.ringQBR, -1, 1)
|
||||
|
||||
// Note: AND3 and OR3 use composition (2 bootstraps) rather than
|
||||
// single-bootstrap with gate constants. Single-bootstrap versions
|
||||
// would require OpenFHE-style gate constant offsets.
|
||||
|
||||
return &BootstrapKey{
|
||||
BRK: brk,
|
||||
KSK: ksk,
|
||||
TestPolyAND: &testPolyAND,
|
||||
TestPolyOR: &testPolyOR,
|
||||
TestPolyXOR: &testPolyXOR,
|
||||
TestPolyNAND: &testPolyNAND,
|
||||
TestPolyNOR: &testPolyNOR,
|
||||
TestPolyXNOR: &testPolyXNOR,
|
||||
TestPolyID: &testPolyID,
|
||||
TestPolyMAJORITY: &testPolyMAJORITY,
|
||||
params: kg.params,
|
||||
BRK: brk,
|
||||
KSK: ksk,
|
||||
TestPolyAND: &testPolyAND,
|
||||
TestPolyOR: &testPolyOR,
|
||||
TestPolyXOR: &testPolyXOR,
|
||||
TestPolyNAND: &testPolyNAND,
|
||||
TestPolyNOR: &testPolyNOR,
|
||||
TestPolyXNOR: &testPolyXNOR,
|
||||
TestPolyID: &testPolyID,
|
||||
TestPolyMAJORITY: &testPolyMAJORITY,
|
||||
TestPolyCMPCOMBINE: &testPolyCMPCOMBINE,
|
||||
params: kg.params,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,564 +0,0 @@
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
// Copyright (c) 2024-2025, Lux Industries Inc
|
||||
//
|
||||
// Optimized Encrypted Comparison for Solidity/EVM
|
||||
//
|
||||
// Key innovation: Kogge-Stone parallel prefix comparison with GPU acceleration
|
||||
// - Traditional: Serial bit-by-bit comparison O(n) depth
|
||||
// - Optimized: Parallel prefix O(log n) depth with early termination hints
|
||||
//
|
||||
// Patent: PAT-FHE-C8 - Encrypted Comparison for Solidity
|
||||
// For enterprise licensing: fhe@lux.network
|
||||
|
||||
#ifndef LUXFHE_ENCRYPTED_COMPARE_H
|
||||
#define LUXFHE_ENCRYPTED_COMPARE_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
|
||||
// =============================================================================
|
||||
// Forward Declarations
|
||||
// =============================================================================
|
||||
|
||||
typedef void* LuxFHEEngine;
|
||||
typedef void* LuxFHECiphertext;
|
||||
typedef void* LuxFHEInteger;
|
||||
typedef void* LuxFHECompareContext;
|
||||
|
||||
// =============================================================================
|
||||
// Comparison Result Types
|
||||
// =============================================================================
|
||||
|
||||
// Comparison operations for Solidity FHE precompile
|
||||
typedef enum {
|
||||
LUXFHE_CMP_LT = 0, // a < b
|
||||
LUXFHE_CMP_LE = 1, // a <= b
|
||||
LUXFHE_CMP_GT = 2, // a > b
|
||||
LUXFHE_CMP_GE = 3, // a >= b
|
||||
LUXFHE_CMP_EQ = 4, // a == b
|
||||
LUXFHE_CMP_NE = 5, // a != b
|
||||
LUXFHE_CMP_MIN = 6, // min(a, b)
|
||||
LUXFHE_CMP_MAX = 7 // max(a, b)
|
||||
} LuxFHECompareOp;
|
||||
|
||||
// Integer bit widths matching Solidity FHE types
|
||||
typedef enum {
|
||||
LUXFHE_UINT4 = 4,
|
||||
LUXFHE_UINT8 = 8,
|
||||
LUXFHE_UINT16 = 16,
|
||||
LUXFHE_UINT32 = 32,
|
||||
LUXFHE_UINT64 = 64,
|
||||
LUXFHE_UINT128 = 128,
|
||||
LUXFHE_UINT160 = 160, // Ethereum address
|
||||
LUXFHE_UINT256 = 256
|
||||
} LuxFHEUintWidth;
|
||||
|
||||
// =============================================================================
|
||||
// Kogge-Stone Parallel Comparison Tree
|
||||
// =============================================================================
|
||||
|
||||
// Configuration for parallel prefix network
|
||||
typedef struct {
|
||||
uint32_t num_bits; // Total bits to compare
|
||||
uint32_t block_size; // Bits per radix block (2 or 4)
|
||||
uint32_t num_stages; // log2(num_bits) stages
|
||||
bool use_gpu; // Enable GPU acceleration
|
||||
bool early_termination; // Enable early termination hints
|
||||
uint32_t batch_size; // Operations to batch for GPU
|
||||
} LuxFHEKoggeStoneConfig;
|
||||
|
||||
// Stage data for Kogge-Stone tree
|
||||
typedef struct {
|
||||
uint32_t stage; // Current stage index
|
||||
uint32_t stride; // Distance between compared elements
|
||||
uint32_t num_ops; // Operations in this stage
|
||||
} LuxFHEKoggeStoneStage;
|
||||
|
||||
// =============================================================================
|
||||
// Parallel Compare Context
|
||||
// =============================================================================
|
||||
|
||||
// Create comparison context with Kogge-Stone configuration
|
||||
LuxFHECompareContext luxfhe_compare_context_create(
|
||||
LuxFHEEngine engine,
|
||||
LuxFHEKoggeStoneConfig config
|
||||
);
|
||||
|
||||
// Free comparison context
|
||||
void luxfhe_compare_context_free(LuxFHECompareContext ctx);
|
||||
|
||||
// Get Kogge-Stone stage info
|
||||
LuxFHEKoggeStoneStage luxfhe_compare_get_stage_info(
|
||||
LuxFHECompareContext ctx,
|
||||
uint32_t stage
|
||||
);
|
||||
|
||||
// =============================================================================
|
||||
// Core Comparison Operations (Solidity FHE Interface)
|
||||
// =============================================================================
|
||||
|
||||
// FHE.lt(euintN a, euintN b) -> ebool
|
||||
// Parallel prefix less-than comparison
|
||||
// Algorithm:
|
||||
// 1. Compute bit differences: d[i] = a[i] XOR b[i]
|
||||
// 2. Kogge-Stone parallel prefix to find MSB difference
|
||||
// 3. Result = b[msb_diff_position]
|
||||
LuxFHECiphertext luxfhe_lt(
|
||||
LuxFHEEngine engine,
|
||||
LuxFHECompareContext ctx,
|
||||
LuxFHEInteger a,
|
||||
LuxFHEInteger b
|
||||
);
|
||||
|
||||
// FHE.le(euintN a, euintN b) -> ebool
|
||||
LuxFHECiphertext luxfhe_le(
|
||||
LuxFHEEngine engine,
|
||||
LuxFHECompareContext ctx,
|
||||
LuxFHEInteger a,
|
||||
LuxFHEInteger b
|
||||
);
|
||||
|
||||
// FHE.gt(euintN a, euintN b) -> ebool
|
||||
LuxFHECiphertext luxfhe_gt(
|
||||
LuxFHEEngine engine,
|
||||
LuxFHECompareContext ctx,
|
||||
LuxFHEInteger a,
|
||||
LuxFHEInteger b
|
||||
);
|
||||
|
||||
// FHE.ge(euintN a, euintN b) -> ebool
|
||||
LuxFHECiphertext luxfhe_ge(
|
||||
LuxFHEEngine engine,
|
||||
LuxFHECompareContext ctx,
|
||||
LuxFHEInteger a,
|
||||
LuxFHEInteger b
|
||||
);
|
||||
|
||||
// FHE.eq(euintN a, euintN b) -> ebool
|
||||
// Parallel XOR-reduction for equality
|
||||
LuxFHECiphertext luxfhe_eq(
|
||||
LuxFHEEngine engine,
|
||||
LuxFHECompareContext ctx,
|
||||
LuxFHEInteger a,
|
||||
LuxFHEInteger b
|
||||
);
|
||||
|
||||
// FHE.ne(euintN a, euintN b) -> ebool
|
||||
LuxFHECiphertext luxfhe_ne(
|
||||
LuxFHEEngine engine,
|
||||
LuxFHECompareContext ctx,
|
||||
LuxFHEInteger a,
|
||||
LuxFHEInteger b
|
||||
);
|
||||
|
||||
// FHE.min(euintN a, euintN b) -> euintN
|
||||
// Returns encrypted minimum using oblivious selection
|
||||
LuxFHEInteger luxfhe_min(
|
||||
LuxFHEEngine engine,
|
||||
LuxFHECompareContext ctx,
|
||||
LuxFHEInteger a,
|
||||
LuxFHEInteger b
|
||||
);
|
||||
|
||||
// FHE.max(euintN a, euintN b) -> euintN
|
||||
LuxFHEInteger luxfhe_max(
|
||||
LuxFHEEngine engine,
|
||||
LuxFHECompareContext ctx,
|
||||
LuxFHEInteger a,
|
||||
LuxFHEInteger b
|
||||
);
|
||||
|
||||
// =============================================================================
|
||||
// Batch Comparison (GPU-Optimized)
|
||||
// =============================================================================
|
||||
|
||||
// Batch comparison for multiple pairs
|
||||
// Enables GPU kernel saturation for high throughput
|
||||
typedef struct {
|
||||
LuxFHEInteger* a_values; // Array of first operands
|
||||
LuxFHEInteger* b_values; // Array of second operands
|
||||
uint32_t count; // Number of comparisons
|
||||
LuxFHECompareOp op; // Comparison operation
|
||||
} LuxFHECompareBatch;
|
||||
|
||||
// Execute batch comparison
|
||||
// Returns array of results (ebool for lt/le/gt/ge/eq/ne, euint for min/max)
|
||||
void* luxfhe_compare_batch(
|
||||
LuxFHEEngine engine,
|
||||
LuxFHECompareContext ctx,
|
||||
LuxFHECompareBatch* batch
|
||||
);
|
||||
|
||||
// Free batch results
|
||||
void luxfhe_compare_batch_free(void* results, uint32_t count, LuxFHECompareOp op);
|
||||
|
||||
// =============================================================================
|
||||
// Scalar Comparison (Plaintext Operand)
|
||||
// =============================================================================
|
||||
|
||||
// FHE.lt(euintN a, uintN b) -> ebool (plaintext right operand)
|
||||
// Optimization: precompute b's bit representation
|
||||
LuxFHECiphertext luxfhe_lt_scalar(
|
||||
LuxFHEEngine engine,
|
||||
LuxFHECompareContext ctx,
|
||||
LuxFHEInteger a,
|
||||
const uint8_t* b_bytes,
|
||||
uint32_t b_len
|
||||
);
|
||||
|
||||
// FHE.eq(euintN a, uintN b) -> ebool (plaintext right operand)
|
||||
LuxFHECiphertext luxfhe_eq_scalar(
|
||||
LuxFHEEngine engine,
|
||||
LuxFHECompareContext ctx,
|
||||
LuxFHEInteger a,
|
||||
const uint8_t* b_bytes,
|
||||
uint32_t b_len
|
||||
);
|
||||
|
||||
// =============================================================================
|
||||
// GPU Kernel Interface (Metal/CUDA)
|
||||
// =============================================================================
|
||||
|
||||
#ifdef LUXFHE_GPU_ENABLED
|
||||
|
||||
// Kogge-Stone parallel prefix kernel configuration
|
||||
typedef struct {
|
||||
uint32_t workgroup_size; // Threads per workgroup
|
||||
uint32_t num_workgroups; // Total workgroups
|
||||
uint32_t shared_mem_bytes; // Shared memory per workgroup
|
||||
} LuxFHEGPUKernelConfig;
|
||||
|
||||
// Get optimal kernel config for hardware
|
||||
LuxFHEGPUKernelConfig luxfhe_compare_get_kernel_config(
|
||||
LuxFHEEngine engine,
|
||||
uint32_t num_bits,
|
||||
uint32_t batch_size
|
||||
);
|
||||
|
||||
// Launch Kogge-Stone parallel prefix kernel
|
||||
// Computes (G, P) pairs: Generate and Propagate signals for comparison
|
||||
// G[i] = a[i] > b[i], P[i] = a[i] == b[i]
|
||||
int luxfhe_gpu_kogge_stone_prefix(
|
||||
LuxFHEEngine engine,
|
||||
LuxFHECompareContext ctx,
|
||||
LuxFHEGPUKernelConfig* config,
|
||||
LuxFHEInteger* a_batch,
|
||||
LuxFHEInteger* b_batch,
|
||||
uint32_t batch_size,
|
||||
LuxFHECiphertext* results
|
||||
);
|
||||
|
||||
// Early termination hint computation
|
||||
// Returns encrypted hint indicating MSB difference position
|
||||
LuxFHECiphertext luxfhe_gpu_early_term_hint(
|
||||
LuxFHEEngine engine,
|
||||
LuxFHECompareContext ctx,
|
||||
LuxFHEInteger a,
|
||||
LuxFHEInteger b
|
||||
);
|
||||
|
||||
#endif // LUXFHE_GPU_ENABLED
|
||||
|
||||
// =============================================================================
|
||||
// Statistics and Profiling
|
||||
// =============================================================================
|
||||
|
||||
typedef struct {
|
||||
uint64_t total_comparisons; // Total comparison operations
|
||||
uint64_t gpu_kernel_launches; // GPU kernel invocations
|
||||
uint64_t early_terminations; // Early termination opportunities
|
||||
double avg_depth_reduction; // Average depth reduction vs serial
|
||||
double avg_latency_us; // Average latency in microseconds
|
||||
double throughput_ops_sec; // Operations per second
|
||||
} LuxFHECompareStats;
|
||||
|
||||
// Get comparison statistics
|
||||
LuxFHECompareStats luxfhe_compare_get_stats(LuxFHECompareContext ctx);
|
||||
|
||||
// Reset statistics
|
||||
void luxfhe_compare_reset_stats(LuxFHECompareContext ctx);
|
||||
|
||||
// =============================================================================
|
||||
// Solidity Interface Specification
|
||||
// =============================================================================
|
||||
|
||||
/*
|
||||
* Solidity FHE Precompile Interface for Comparison Operations
|
||||
*
|
||||
* Address: 0x0100 (FHE Precompile Base)
|
||||
*
|
||||
* Function Selectors:
|
||||
* lt(bytes32 a, bytes32 b) -> 0x1234... // Returns encrypted bool
|
||||
* le(bytes32 a, bytes32 b) -> 0x2345...
|
||||
* gt(bytes32 a, bytes32 b) -> 0x3456...
|
||||
* ge(bytes32 a, bytes32 b) -> 0x4567...
|
||||
* eq(bytes32 a, bytes32 b) -> 0x5678...
|
||||
* ne(bytes32 a, bytes32 b) -> 0x6789...
|
||||
* min(bytes32 a, bytes32 b) -> 0x789a... // Returns encrypted uint
|
||||
* max(bytes32 a, bytes32 b) -> 0x89ab...
|
||||
*
|
||||
* Input Format:
|
||||
* - bytes32 handle: Ciphertext handle in global ciphertext store
|
||||
* - Type info encoded in handle's upper bits
|
||||
*
|
||||
* Gas Costs (approximate):
|
||||
* - lt/gt/le/ge: 50000 + 500 * log2(bit_width) gas
|
||||
* - eq/ne: 30000 + 300 * log2(bit_width) gas
|
||||
* - min/max: 80000 + 800 * log2(bit_width) gas
|
||||
*
|
||||
* Example Solidity Usage:
|
||||
*
|
||||
* import "fhe/FHE.sol";
|
||||
*
|
||||
* contract Auction {
|
||||
* euint256 highestBid;
|
||||
* eaddress highestBidder;
|
||||
*
|
||||
* function bid(einput encryptedBid, bytes calldata proof) external {
|
||||
* euint256 bidAmount = FHE.asEuint256(encryptedBid, proof);
|
||||
*
|
||||
* // Encrypted comparison: is new bid higher?
|
||||
* ebool isHigher = FHE.gt(bidAmount, highestBid);
|
||||
*
|
||||
* // Oblivious selection: update if higher
|
||||
* highestBid = FHE.select(isHigher, bidAmount, highestBid);
|
||||
* highestBidder = FHE.select(isHigher,
|
||||
* FHE.asEaddress(msg.sender),
|
||||
* highestBidder);
|
||||
* }
|
||||
* }
|
||||
*/
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
// =============================================================================
|
||||
// C++ API (when compiled as C++)
|
||||
// =============================================================================
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
#include <vector>
|
||||
#include <memory>
|
||||
#include <functional>
|
||||
|
||||
namespace luxfhe {
|
||||
namespace compare {
|
||||
|
||||
// Encrypted bit type (forward declaration)
|
||||
class EncryptedBit;
|
||||
class EncryptedInteger;
|
||||
|
||||
// =============================================================================
|
||||
// Kogge-Stone Parallel Prefix Tree
|
||||
// =============================================================================
|
||||
|
||||
// Generate-Propagate pair for comparison
|
||||
// G = a > b (generate: this position determines result)
|
||||
// P = a == b (propagate: defer to higher position)
|
||||
struct GPPair {
|
||||
std::shared_ptr<EncryptedBit> G; // Generate signal
|
||||
std::shared_ptr<EncryptedBit> P; // Propagate signal
|
||||
};
|
||||
|
||||
// Kogge-Stone operator: combines two GP pairs
|
||||
// (G1, P1) o (G0, P0) = (G1 OR (P1 AND G0), P1 AND P0)
|
||||
class KoggeStoneOp {
|
||||
public:
|
||||
virtual ~KoggeStoneOp() = default;
|
||||
|
||||
// Combine two GP pairs homomorphically
|
||||
virtual GPPair combine(const GPPair& high, const GPPair& low) = 0;
|
||||
};
|
||||
|
||||
// =============================================================================
|
||||
// ParallelCompare Class
|
||||
// =============================================================================
|
||||
|
||||
class ParallelCompare {
|
||||
public:
|
||||
// Configuration
|
||||
struct Config {
|
||||
uint32_t num_bits = 256;
|
||||
uint32_t block_size = 4;
|
||||
bool use_gpu = true;
|
||||
bool early_termination = true;
|
||||
uint32_t batch_size = 1024;
|
||||
};
|
||||
|
||||
// Constructor
|
||||
explicit ParallelCompare(Config config);
|
||||
~ParallelCompare();
|
||||
|
||||
// Disable copy, enable move
|
||||
ParallelCompare(const ParallelCompare&) = delete;
|
||||
ParallelCompare& operator=(const ParallelCompare&) = delete;
|
||||
ParallelCompare(ParallelCompare&&) noexcept;
|
||||
ParallelCompare& operator=(ParallelCompare&&) noexcept;
|
||||
|
||||
// =========================================================================
|
||||
// Core Comparison (Kogge-Stone Algorithm)
|
||||
// =========================================================================
|
||||
|
||||
// Less-than comparison using parallel prefix
|
||||
// Depth: O(log n) vs O(n) for serial
|
||||
std::shared_ptr<EncryptedBit> lessThan(
|
||||
const EncryptedInteger& a,
|
||||
const EncryptedInteger& b
|
||||
);
|
||||
|
||||
// Equality using parallel XOR-reduction
|
||||
std::shared_ptr<EncryptedBit> equal(
|
||||
const EncryptedInteger& a,
|
||||
const EncryptedInteger& b
|
||||
);
|
||||
|
||||
// =========================================================================
|
||||
// Derived Operations
|
||||
// =========================================================================
|
||||
|
||||
std::shared_ptr<EncryptedBit> lessEqual(
|
||||
const EncryptedInteger& a,
|
||||
const EncryptedInteger& b
|
||||
);
|
||||
|
||||
std::shared_ptr<EncryptedBit> greaterThan(
|
||||
const EncryptedInteger& a,
|
||||
const EncryptedInteger& b
|
||||
);
|
||||
|
||||
std::shared_ptr<EncryptedBit> greaterEqual(
|
||||
const EncryptedInteger& a,
|
||||
const EncryptedInteger& b
|
||||
);
|
||||
|
||||
std::shared_ptr<EncryptedBit> notEqual(
|
||||
const EncryptedInteger& a,
|
||||
const EncryptedInteger& b
|
||||
);
|
||||
|
||||
// =========================================================================
|
||||
// Selection Operations
|
||||
// =========================================================================
|
||||
|
||||
// Oblivious minimum: returns a if a < b, else b
|
||||
std::shared_ptr<EncryptedInteger> min(
|
||||
const EncryptedInteger& a,
|
||||
const EncryptedInteger& b
|
||||
);
|
||||
|
||||
// Oblivious maximum
|
||||
std::shared_ptr<EncryptedInteger> max(
|
||||
const EncryptedInteger& a,
|
||||
const EncryptedInteger& b
|
||||
);
|
||||
|
||||
// Oblivious selection: returns a if cond is true, else b
|
||||
std::shared_ptr<EncryptedInteger> select(
|
||||
const EncryptedBit& cond,
|
||||
const EncryptedInteger& a,
|
||||
const EncryptedInteger& b
|
||||
);
|
||||
|
||||
// =========================================================================
|
||||
// Batch Operations (GPU-Optimized)
|
||||
// =========================================================================
|
||||
|
||||
std::vector<std::shared_ptr<EncryptedBit>> batchLessThan(
|
||||
const std::vector<EncryptedInteger>& a_vec,
|
||||
const std::vector<EncryptedInteger>& b_vec
|
||||
);
|
||||
|
||||
std::vector<std::shared_ptr<EncryptedBit>> batchEqual(
|
||||
const std::vector<EncryptedInteger>& a_vec,
|
||||
const std::vector<EncryptedInteger>& b_vec
|
||||
);
|
||||
|
||||
// =========================================================================
|
||||
// Scalar Comparisons (Optimized)
|
||||
// =========================================================================
|
||||
|
||||
std::shared_ptr<EncryptedBit> lessThanScalar(
|
||||
const EncryptedInteger& a,
|
||||
const std::vector<uint8_t>& b_plaintext
|
||||
);
|
||||
|
||||
std::shared_ptr<EncryptedBit> equalScalar(
|
||||
const EncryptedInteger& a,
|
||||
const std::vector<uint8_t>& b_plaintext
|
||||
);
|
||||
|
||||
private:
|
||||
class Impl;
|
||||
std::unique_ptr<Impl> impl_;
|
||||
|
||||
// Kogge-Stone tree building
|
||||
std::vector<GPPair> buildGPPairs(
|
||||
const EncryptedInteger& a,
|
||||
const EncryptedInteger& b
|
||||
);
|
||||
|
||||
// Parallel prefix computation
|
||||
GPPair parallelPrefix(const std::vector<GPPair>& gp_pairs);
|
||||
|
||||
// GPU kernel dispatch
|
||||
void dispatchGPUKernel(
|
||||
const std::vector<GPPair>& gp_pairs,
|
||||
uint32_t stage,
|
||||
uint32_t stride
|
||||
);
|
||||
};
|
||||
|
||||
// =============================================================================
|
||||
// GPU Kernel Launcher
|
||||
// =============================================================================
|
||||
|
||||
class GPUCompareKernel {
|
||||
public:
|
||||
struct LaunchParams {
|
||||
uint32_t workgroup_size = 256;
|
||||
uint32_t num_workgroups = 0; // 0 = auto
|
||||
size_t shared_mem_bytes = 0;
|
||||
};
|
||||
|
||||
virtual ~GPUCompareKernel() = default;
|
||||
|
||||
// Launch Kogge-Stone stage kernel
|
||||
// Each stage computes GP pairs with distance 2^stage
|
||||
virtual void launchKoggeStoneStage(
|
||||
const LaunchParams& params,
|
||||
std::vector<GPPair>& gp_pairs,
|
||||
uint32_t stage
|
||||
) = 0;
|
||||
|
||||
// Launch XOR-reduction kernel for equality
|
||||
virtual void launchXorReduction(
|
||||
const LaunchParams& params,
|
||||
std::vector<std::shared_ptr<EncryptedBit>>& xor_bits
|
||||
) = 0;
|
||||
|
||||
// Launch oblivious selection kernel
|
||||
virtual void launchSelect(
|
||||
const LaunchParams& params,
|
||||
const EncryptedBit& cond,
|
||||
const EncryptedInteger& a,
|
||||
const EncryptedInteger& b,
|
||||
EncryptedInteger& result
|
||||
) = 0;
|
||||
};
|
||||
|
||||
// Factory for platform-specific kernel
|
||||
std::unique_ptr<GPUCompareKernel> createGPUCompareKernel();
|
||||
|
||||
} // namespace compare
|
||||
} // namespace luxfhe
|
||||
|
||||
#endif // __cplusplus
|
||||
|
||||
#endif // LUXFHE_ENCRYPTED_COMPARE_H
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,242 +0,0 @@
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
// Copyright (c) 2025, Lux Industries Inc
|
||||
//
|
||||
// C bridge header for OpenFHE BinFHE (TFHE) operations
|
||||
// This header defines the C interface that Go calls via CGO
|
||||
|
||||
#ifndef TFHE_BRIDGE_H
|
||||
#define TFHE_BRIDGE_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
|
||||
// Opaque handle types
|
||||
typedef void* TfheContext;
|
||||
typedef void* TfheSecretKey;
|
||||
typedef void* TfhePublicKey;
|
||||
typedef void* TfheCiphertext;
|
||||
typedef void* TfheBootstrapKey;
|
||||
|
||||
// Security levels matching OpenFHE BINFHE_PARAMSET
|
||||
typedef enum {
|
||||
TFHE_TOY = 0, // ~16-bit security (testing only)
|
||||
TFHE_STD128 = 1, // 128-bit security (CGGI/GINX)
|
||||
TFHE_STD128_AP = 2, // 128-bit security (AP variant)
|
||||
TFHE_STD128_LMKCDEY = 3, // 128-bit security (LMKCDEY - fastest)
|
||||
TFHE_STD192 = 4, // 192-bit security
|
||||
TFHE_STD256 = 5 // 256-bit security
|
||||
} TfheSecurityLevel;
|
||||
|
||||
// Bootstrapping method
|
||||
typedef enum {
|
||||
TFHE_METHOD_GINX = 0, // GINX (default)
|
||||
TFHE_METHOD_AP = 1, // AP variant
|
||||
TFHE_METHOD_LMKCDEY = 2 // LMKCDEY (fastest)
|
||||
} TfheMethod;
|
||||
|
||||
// =============================================================================
|
||||
// Context Management
|
||||
// =============================================================================
|
||||
|
||||
// Create a new TFHE context with specified security level and method
|
||||
TfheContext tfhe_context_new(TfheSecurityLevel level, TfheMethod method);
|
||||
|
||||
// Free a TFHE context
|
||||
void tfhe_context_free(TfheContext ctx);
|
||||
|
||||
// Get version of the bridge library
|
||||
uint32_t tfhe_version(void);
|
||||
|
||||
// =============================================================================
|
||||
// Key Generation
|
||||
// =============================================================================
|
||||
|
||||
// Generate a secret key
|
||||
TfheSecretKey tfhe_keygen(TfheContext ctx);
|
||||
|
||||
// Free a secret key
|
||||
void tfhe_secretkey_free(TfheSecretKey sk);
|
||||
|
||||
// Generate bootstrap key (required for gate evaluation)
|
||||
int tfhe_bootstrap_keygen(TfheContext ctx, TfheSecretKey sk);
|
||||
|
||||
// Generate key switching key
|
||||
int tfhe_keyswitch_keygen(TfheContext ctx, TfheSecretKey sk);
|
||||
|
||||
// Check if bootstrap key is generated
|
||||
bool tfhe_has_bootstrap_key(TfheContext ctx);
|
||||
|
||||
// =============================================================================
|
||||
// Public Key Generation
|
||||
// =============================================================================
|
||||
|
||||
// Generate a public key from secret key
|
||||
TfhePublicKey tfhe_public_keygen(TfheContext ctx, TfheSecretKey sk);
|
||||
|
||||
// Free a public key
|
||||
void tfhe_public_key_free(TfhePublicKey pk);
|
||||
|
||||
// Serialize public key to bytes
|
||||
int tfhe_publickey_serialize(TfhePublicKey pk, uint8_t** out, size_t* out_len);
|
||||
|
||||
// Deserialize public key from bytes
|
||||
TfhePublicKey tfhe_publickey_deserialize(TfheContext ctx, const uint8_t* data, size_t len);
|
||||
|
||||
// =============================================================================
|
||||
// Encryption / Decryption (Boolean)
|
||||
// =============================================================================
|
||||
|
||||
// Encrypt a boolean value (0 or 1)
|
||||
TfheCiphertext tfhe_encrypt(TfheContext ctx, TfheSecretKey sk, int value);
|
||||
|
||||
// Encrypt a bit (alias for tfhe_encrypt)
|
||||
TfheCiphertext tfhe_encrypt_bit(TfheContext ctx, TfheSecretKey sk, int value);
|
||||
|
||||
// Encrypt a bit with public key
|
||||
TfheCiphertext tfhe_encrypt_bit_public(TfheContext ctx, TfhePublicKey pk, int value);
|
||||
|
||||
// Decrypt a ciphertext to boolean
|
||||
int tfhe_decrypt(TfheContext ctx, TfheSecretKey sk, TfheCiphertext ct);
|
||||
|
||||
// Decrypt a bit (alias for tfhe_decrypt)
|
||||
int tfhe_decrypt_bit(TfheContext ctx, TfheSecretKey sk, TfheCiphertext ct);
|
||||
|
||||
// Free a ciphertext
|
||||
void tfhe_ciphertext_free(TfheCiphertext ct);
|
||||
|
||||
// Free a secret key (alias for backward compatibility)
|
||||
void tfhe_secret_key_free(TfheSecretKey sk);
|
||||
|
||||
// Clone a ciphertext
|
||||
TfheCiphertext tfhe_ciphertext_clone(TfheCiphertext ct);
|
||||
|
||||
// =============================================================================
|
||||
// Boolean Gates (with bootstrapping)
|
||||
// =============================================================================
|
||||
|
||||
TfheCiphertext tfhe_and(TfheContext ctx, TfheCiphertext ct1, TfheCiphertext ct2);
|
||||
TfheCiphertext tfhe_or(TfheContext ctx, TfheCiphertext ct1, TfheCiphertext ct2);
|
||||
TfheCiphertext tfhe_xor(TfheContext ctx, TfheCiphertext ct1, TfheCiphertext ct2);
|
||||
TfheCiphertext tfhe_nand(TfheContext ctx, TfheCiphertext ct1, TfheCiphertext ct2);
|
||||
TfheCiphertext tfhe_nor(TfheContext ctx, TfheCiphertext ct1, TfheCiphertext ct2);
|
||||
TfheCiphertext tfhe_xnor(TfheContext ctx, TfheCiphertext ct1, TfheCiphertext ct2);
|
||||
TfheCiphertext tfhe_not(TfheContext ctx, TfheCiphertext ct);
|
||||
TfheCiphertext tfhe_mux(TfheContext ctx, TfheCiphertext sel, TfheCiphertext ct1, TfheCiphertext ct2);
|
||||
|
||||
// =============================================================================
|
||||
// Integer Operations (radix representation)
|
||||
// =============================================================================
|
||||
|
||||
// Integer ciphertext handle
|
||||
typedef void* TfheInteger;
|
||||
|
||||
// Integer types
|
||||
typedef enum {
|
||||
TFHE_UINT4 = 4,
|
||||
TFHE_UINT8 = 8,
|
||||
TFHE_UINT16 = 16,
|
||||
TFHE_UINT32 = 32,
|
||||
TFHE_UINT64 = 64,
|
||||
TFHE_UINT128 = 128,
|
||||
TFHE_UINT160 = 160,
|
||||
TFHE_UINT256 = 256
|
||||
} TfheIntType;
|
||||
|
||||
// Encrypt an integer
|
||||
TfheInteger tfhe_encrypt_integer(TfheContext ctx, TfheSecretKey sk, int64_t value, int bitLen);
|
||||
|
||||
// Encrypt an integer with public key
|
||||
TfheInteger tfhe_encrypt_integer_public(TfheContext ctx, TfhePublicKey pk, int64_t value, int bitLen);
|
||||
|
||||
// Decrypt an integer
|
||||
int64_t tfhe_decrypt_integer(TfheContext ctx, TfheSecretKey sk, TfheInteger ct);
|
||||
|
||||
// Free an integer ciphertext
|
||||
void tfhe_integer_free(TfheInteger ct);
|
||||
|
||||
// Clone an integer ciphertext
|
||||
TfheInteger tfhe_integer_clone(TfheInteger ct);
|
||||
|
||||
// Get the type of an integer ciphertext
|
||||
TfheIntType tfhe_integer_type(TfheInteger ct);
|
||||
|
||||
// =============================================================================
|
||||
// Integer Arithmetic
|
||||
// =============================================================================
|
||||
|
||||
TfheInteger tfhe_add(TfheContext ctx, TfheInteger a, TfheInteger b);
|
||||
TfheInteger tfhe_sub(TfheContext ctx, TfheInteger a, TfheInteger b);
|
||||
TfheInteger tfhe_neg(TfheContext ctx, TfheInteger a);
|
||||
TfheInteger tfhe_add_scalar(TfheContext ctx, TfheInteger a, int64_t scalar);
|
||||
TfheInteger tfhe_sub_scalar(TfheContext ctx, TfheInteger a, int64_t scalar);
|
||||
TfheInteger tfhe_mul_scalar(TfheContext ctx, TfheInteger a, int64_t scalar);
|
||||
|
||||
// =============================================================================
|
||||
// Integer Comparisons
|
||||
// =============================================================================
|
||||
|
||||
TfheCiphertext tfhe_eq(TfheContext ctx, TfheInteger a, TfheInteger b);
|
||||
TfheCiphertext tfhe_ne(TfheContext ctx, TfheInteger a, TfheInteger b);
|
||||
TfheCiphertext tfhe_lt(TfheContext ctx, TfheInteger a, TfheInteger b);
|
||||
TfheCiphertext tfhe_le(TfheContext ctx, TfheInteger a, TfheInteger b);
|
||||
TfheCiphertext tfhe_gt(TfheContext ctx, TfheInteger a, TfheInteger b);
|
||||
TfheCiphertext tfhe_ge(TfheContext ctx, TfheInteger a, TfheInteger b);
|
||||
TfheInteger tfhe_min(TfheContext ctx, TfheInteger a, TfheInteger b);
|
||||
TfheInteger tfhe_max(TfheContext ctx, TfheInteger a, TfheInteger b);
|
||||
|
||||
// =============================================================================
|
||||
// Integer Bitwise Operations
|
||||
// =============================================================================
|
||||
|
||||
TfheInteger tfhe_bitwise_and(TfheContext ctx, TfheInteger a, TfheInteger b);
|
||||
TfheInteger tfhe_bitwise_or(TfheContext ctx, TfheInteger a, TfheInteger b);
|
||||
TfheInteger tfhe_bitwise_xor(TfheContext ctx, TfheInteger a, TfheInteger b);
|
||||
TfheInteger tfhe_bitwise_not(TfheContext ctx, TfheInteger a);
|
||||
TfheInteger tfhe_shl(TfheContext ctx, TfheInteger a, int bits);
|
||||
TfheInteger tfhe_shr(TfheContext ctx, TfheInteger a, int bits);
|
||||
|
||||
// =============================================================================
|
||||
// Control Flow
|
||||
// =============================================================================
|
||||
|
||||
TfheInteger tfhe_select(TfheContext ctx, TfheCiphertext cond, TfheInteger if_true, TfheInteger if_false);
|
||||
TfheInteger tfhe_cast_to(TfheContext ctx, TfheInteger a, int target_bitlen);
|
||||
|
||||
// =============================================================================
|
||||
// Serialization
|
||||
// =============================================================================
|
||||
|
||||
// Serialize ciphertext to bytes (returns malloc'd buffer, caller must free)
|
||||
uint8_t* tfhe_serialize_ciphertext(TfheContext ctx, TfheCiphertext ct, size_t* out_len);
|
||||
|
||||
// Deserialize ciphertext from bytes
|
||||
TfheCiphertext tfhe_deserialize_ciphertext(TfheContext ctx, const uint8_t* data, size_t len);
|
||||
|
||||
// Serialize secret key to bytes (returns malloc'd buffer, caller must free)
|
||||
uint8_t* tfhe_serialize_secret_key(TfheContext ctx, TfheSecretKey sk, size_t* out_len);
|
||||
|
||||
// Deserialize secret key from bytes
|
||||
TfheSecretKey tfhe_deserialize_secret_key(TfheContext ctx, const uint8_t* data, size_t len);
|
||||
|
||||
// Serialize public key to bytes (returns malloc'd buffer, caller must free)
|
||||
uint8_t* tfhe_serialize_public_key(TfheContext ctx, TfhePublicKey pk, size_t* out_len);
|
||||
|
||||
// Deserialize public key from bytes
|
||||
TfhePublicKey tfhe_deserialize_public_key(TfheContext ctx, const uint8_t* data, size_t len);
|
||||
|
||||
// Serialize integer ciphertext to bytes (returns malloc'd buffer, caller must free)
|
||||
uint8_t* tfhe_serialize_integer(TfheContext ctx, TfheInteger ct, size_t* out_len);
|
||||
|
||||
// Deserialize integer ciphertext from bytes
|
||||
TfheInteger tfhe_deserialize_integer(TfheContext ctx, const uint8_t* data, size_t len);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // TFHE_BRIDGE_H
|
||||
-1164
File diff suppressed because it is too large
Load Diff
-754
@@ -1,754 +0,0 @@
|
||||
//go:build cgo
|
||||
|
||||
// Package gpu provides GPU-accelerated tensor operations for FHE workloads.
|
||||
// These operations are needed for GPU-accelerated TFHE bootstrapping.
|
||||
//
|
||||
// Copyright (c) 2024-2025 Lux Industries Inc.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
package gpu
|
||||
|
||||
/*
|
||||
#cgo pkg-config: lux-gpu
|
||||
#cgo CFLAGS: -I${SRCDIR}/../../cpp/gpu
|
||||
#include <lux/gpu/gpu.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
// Short aliases for Go code readability (hides lux_gpu_ prefix)
|
||||
|
||||
// Types
|
||||
#define GPU LuxGPU
|
||||
#define Tensor LuxTensor
|
||||
#define DType LuxDType
|
||||
|
||||
// Tensor creation
|
||||
#define tensor_create lux_gpu_tensor_create
|
||||
#define tensor_full lux_gpu_tensor_full
|
||||
#define tensor_data lux_gpu_tensor_data
|
||||
|
||||
// Tensor operations (arithmetic)
|
||||
#define gpu_add lux_gpu_add
|
||||
#define gpu_subtract lux_gpu_subtract
|
||||
#define gpu_multiply lux_gpu_multiply
|
||||
#define gpu_divide lux_gpu_divide
|
||||
#define gpu_remainder lux_gpu_remainder
|
||||
#define gpu_floor lux_gpu_floor
|
||||
#define gpu_round lux_gpu_round
|
||||
#define gpu_right_shift lux_gpu_right_shift
|
||||
#define gpu_astype lux_gpu_astype
|
||||
|
||||
// Tensor operations (shape manipulation)
|
||||
#define gpu_reshape lux_gpu_reshape
|
||||
#define gpu_squeeze lux_gpu_squeeze
|
||||
#define gpu_broadcast lux_gpu_broadcast
|
||||
#define gpu_slice lux_gpu_slice
|
||||
#define gpu_take lux_gpu_take
|
||||
#define gpu_take_along_axis lux_gpu_take_along_axis
|
||||
#define gpu_concatenate lux_gpu_concatenate
|
||||
|
||||
// Tensor operations (comparison)
|
||||
#define gpu_less lux_gpu_less
|
||||
#define gpu_greater lux_gpu_greater
|
||||
#define gpu_less_equal lux_gpu_less_equal
|
||||
#define gpu_greater_equal lux_gpu_greater_equal
|
||||
#define gpu_where lux_gpu_where
|
||||
|
||||
// Tensor operations (linear algebra & reduction)
|
||||
#define gpu_matmul lux_gpu_matmul
|
||||
#define gpu_sum lux_gpu_sum
|
||||
#define gpu_mean lux_gpu_mean
|
||||
#define gpu_max lux_gpu_max
|
||||
#define gpu_min lux_gpu_min
|
||||
*/
|
||||
import "C"
|
||||
import "unsafe"
|
||||
|
||||
// intsToCInts converts a Go []int slice to []C.int for CGO calls
|
||||
// Go int is 64-bit on 64-bit systems, C int is 32-bit
|
||||
func intsToCInts(ints []int) []C.int {
|
||||
cInts := make([]C.int, len(ints))
|
||||
for i, v := range ints {
|
||||
cInts[i] = C.int(v)
|
||||
}
|
||||
return cInts
|
||||
}
|
||||
|
||||
// SliceArg represents slicing arguments for Slice operation
|
||||
type SliceArg struct {
|
||||
Start int
|
||||
Stop int
|
||||
}
|
||||
|
||||
// Add performs element-wise addition: a + b
|
||||
func Add(a, b *Array) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
tensor := C.gpu_add(DefaultContext.gpu, a.tensor, b.tensor)
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: a.shape,
|
||||
dtype: a.dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Subtract performs element-wise subtraction: a - b
|
||||
func Subtract(a, b *Array) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
tensor := C.gpu_subtract(DefaultContext.gpu, a.tensor, b.tensor)
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: a.shape,
|
||||
dtype: a.dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Multiply performs element-wise multiplication: a * b
|
||||
func Multiply(a, b *Array) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
tensor := C.gpu_multiply(DefaultContext.gpu, a.tensor, b.tensor)
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: a.shape,
|
||||
dtype: a.dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Divide performs element-wise division: a / b
|
||||
func Divide(a, b *Array) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
tensor := C.gpu_divide(DefaultContext.gpu, a.tensor, b.tensor)
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: a.shape,
|
||||
dtype: a.dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Remainder computes element-wise remainder: a % b
|
||||
func Remainder(a, b *Array) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
tensor := C.gpu_remainder(DefaultContext.gpu, a.tensor, b.tensor)
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: a.shape,
|
||||
dtype: a.dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Floor computes element-wise floor
|
||||
func Floor(a *Array) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
tensor := C.gpu_floor(DefaultContext.gpu, a.tensor)
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: a.shape,
|
||||
dtype: a.dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Round rounds elements to nearest integer
|
||||
func Round(a *Array) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
tensor := C.gpu_round(DefaultContext.gpu, a.tensor)
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: a.shape,
|
||||
dtype: a.dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// RightShift performs element-wise right shift: a >> bits
|
||||
func RightShift(a *Array, bits int) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
tensor := C.gpu_right_shift(DefaultContext.gpu, a.tensor, C.int(bits))
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: a.shape,
|
||||
dtype: a.dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// AsType casts array to specified dtype
|
||||
func AsType(a *Array, dtype Dtype) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
tensor := C.gpu_astype(DefaultContext.gpu, a.tensor, dtypeToC(dtype))
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: a.shape,
|
||||
dtype: dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Full creates an array filled with a constant value
|
||||
func Full(shape []int, value interface{}, dtype Dtype) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
var fval float32
|
||||
switch v := value.(type) {
|
||||
case float64:
|
||||
fval = float32(v)
|
||||
case float32:
|
||||
fval = v
|
||||
case int:
|
||||
fval = float32(v)
|
||||
case int32:
|
||||
fval = float32(v)
|
||||
case int64:
|
||||
fval = float32(v)
|
||||
case uint64:
|
||||
fval = float32(v)
|
||||
}
|
||||
|
||||
cShape := intsToCInts(shape)
|
||||
tensor := C.tensor_full(
|
||||
DefaultContext.gpu,
|
||||
&cShape[0],
|
||||
C.int(len(shape)),
|
||||
C.float(fval),
|
||||
dtypeToC(dtype),
|
||||
)
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: shape,
|
||||
dtype: dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Reshape reshapes array to new shape
|
||||
func Reshape(a *Array, shape []int) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
cShape := intsToCInts(shape)
|
||||
tensor := C.gpu_reshape(DefaultContext.gpu, a.tensor, &cShape[0], C.int(len(shape)))
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: shape,
|
||||
dtype: a.dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Squeeze removes dimension of size 1 at specified axis
|
||||
func Squeeze(a *Array, axis int) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
tensor := C.gpu_squeeze(DefaultContext.gpu, a.tensor, C.int(axis))
|
||||
|
||||
// Calculate new shape
|
||||
newShape := make([]int, 0, len(a.shape)-1)
|
||||
for i, s := range a.shape {
|
||||
if i != axis && (axis >= 0 || i != len(a.shape)+axis) {
|
||||
newShape = append(newShape, s)
|
||||
}
|
||||
}
|
||||
if len(newShape) == 0 {
|
||||
newShape = []int{1}
|
||||
}
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: newShape,
|
||||
dtype: a.dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Broadcast broadcasts array to specified shape
|
||||
func Broadcast(a *Array, shape []int) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
cShape := intsToCInts(shape)
|
||||
tensor := C.gpu_broadcast(DefaultContext.gpu, a.tensor, &cShape[0], C.int(len(shape)))
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: shape,
|
||||
dtype: a.dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Slice extracts a slice from the array
|
||||
func Slice(a *Array, args []SliceArg) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
ndim := len(args)
|
||||
starts := make([]int, ndim)
|
||||
stops := make([]int, ndim)
|
||||
steps := make([]int, ndim)
|
||||
|
||||
for i, arg := range args {
|
||||
starts[i] = arg.Start
|
||||
stops[i] = arg.Stop
|
||||
steps[i] = 1 // Default step
|
||||
}
|
||||
|
||||
cStarts := intsToCInts(starts)
|
||||
cStops := intsToCInts(stops)
|
||||
cSteps := intsToCInts(steps)
|
||||
tensor := C.gpu_slice(DefaultContext.gpu, a.tensor, &cStarts[0], &cStops[0], &cSteps[0], C.int(ndim))
|
||||
|
||||
// Calculate new shape
|
||||
newShape := make([]int, ndim)
|
||||
for i := 0; i < ndim; i++ {
|
||||
newShape[i] = stops[i] - starts[i]
|
||||
}
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: newShape,
|
||||
dtype: a.dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Take gathers elements along an axis
|
||||
func Take(a *Array, indices *Array, axis int) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
tensor := C.gpu_take(DefaultContext.gpu, a.tensor, indices.tensor, C.int(axis))
|
||||
|
||||
// Shape depends on indices shape replacing the axis dimension
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: indices.shape,
|
||||
dtype: a.dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// TakeAlongAxis gathers values along an axis using indices of the same shape
|
||||
func TakeAlongAxis(a, indices *Array, axis int) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
tensor := C.gpu_take_along_axis(DefaultContext.gpu, a.tensor, indices.tensor, C.int(axis))
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: indices.shape,
|
||||
dtype: a.dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Concatenate concatenates arrays along an axis
|
||||
func Concatenate(arrays []Array, axis int) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
tensors := make([]*C.Tensor, len(arrays))
|
||||
for i := range arrays {
|
||||
tensors[i] = arrays[i].tensor
|
||||
}
|
||||
|
||||
tensor := C.gpu_concatenate(DefaultContext.gpu, &tensors[0], C.int(len(arrays)), C.int(axis))
|
||||
|
||||
// Calculate output shape
|
||||
newShape := make([]int, len(arrays[0].shape))
|
||||
copy(newShape, arrays[0].shape)
|
||||
for i := 1; i < len(arrays); i++ {
|
||||
newShape[axis] += arrays[i].shape[axis]
|
||||
}
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: newShape,
|
||||
dtype: arrays[0].dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Less compares element-wise: a < b
|
||||
func Less(a, b *Array) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
tensor := C.gpu_less(DefaultContext.gpu, a.tensor, b.tensor)
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: a.shape,
|
||||
dtype: Bool,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Greater compares element-wise: a > b
|
||||
func Greater(a, b *Array) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
tensor := C.gpu_greater(DefaultContext.gpu, a.tensor, b.tensor)
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: a.shape,
|
||||
dtype: Bool,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// LessEqual compares element-wise: a <= b
|
||||
func LessEqual(a, b *Array) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
tensor := C.gpu_less_equal(DefaultContext.gpu, a.tensor, b.tensor)
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: a.shape,
|
||||
dtype: Bool,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// GreaterEqual compares element-wise: a >= b
|
||||
func GreaterEqual(a, b *Array) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
tensor := C.gpu_greater_equal(DefaultContext.gpu, a.tensor, b.tensor)
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: a.shape,
|
||||
dtype: Bool,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Where selects elements based on condition: cond ? a : b
|
||||
func Where(cond, a, b *Array) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
tensor := C.gpu_where(DefaultContext.gpu, cond.tensor, a.tensor, b.tensor)
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: a.shape,
|
||||
dtype: a.dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Matmul performs matrix multiplication
|
||||
func Matmul(a, b *Array) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
tensor := C.gpu_matmul(DefaultContext.gpu, a.tensor, b.tensor)
|
||||
|
||||
// Calculate output shape for matmul
|
||||
newShape := make([]int, len(a.shape))
|
||||
copy(newShape, a.shape)
|
||||
if len(b.shape) > 1 {
|
||||
newShape[len(newShape)-1] = b.shape[len(b.shape)-1]
|
||||
}
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: newShape,
|
||||
dtype: a.dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Sum reduces array by summing along specified axes
|
||||
func Sum(a *Array, axes []int) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
cAxes := intsToCInts(axes)
|
||||
var tensor *C.Tensor
|
||||
if len(axes) > 0 {
|
||||
tensor = C.gpu_sum(DefaultContext.gpu, a.tensor, &cAxes[0], C.int(len(axes)))
|
||||
} else {
|
||||
tensor = C.gpu_sum(DefaultContext.gpu, a.tensor, nil, 0)
|
||||
}
|
||||
|
||||
// Calculate reduced shape
|
||||
newShape := make([]int, 0)
|
||||
for i, s := range a.shape {
|
||||
keep := true
|
||||
for _, ax := range axes {
|
||||
if i == ax {
|
||||
keep = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if keep {
|
||||
newShape = append(newShape, s)
|
||||
}
|
||||
}
|
||||
if len(newShape) == 0 {
|
||||
newShape = []int{1}
|
||||
}
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: newShape,
|
||||
dtype: a.dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Mean reduces array by computing mean along specified axes
|
||||
func Mean(a *Array, axes []int) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
cAxes := intsToCInts(axes)
|
||||
var tensor *C.Tensor
|
||||
if len(axes) > 0 {
|
||||
tensor = C.gpu_mean(DefaultContext.gpu, a.tensor, &cAxes[0], C.int(len(axes)))
|
||||
} else {
|
||||
tensor = C.gpu_mean(DefaultContext.gpu, a.tensor, nil, 0)
|
||||
}
|
||||
|
||||
// Calculate reduced shape
|
||||
newShape := make([]int, 0)
|
||||
for i, s := range a.shape {
|
||||
keep := true
|
||||
for _, ax := range axes {
|
||||
if i == ax {
|
||||
keep = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if keep {
|
||||
newShape = append(newShape, s)
|
||||
}
|
||||
}
|
||||
if len(newShape) == 0 {
|
||||
newShape = []int{1}
|
||||
}
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: newShape,
|
||||
dtype: a.dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Max reduces array by computing max along specified axes
|
||||
func Max(a *Array, axes []int) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
cAxes := intsToCInts(axes)
|
||||
var tensor *C.Tensor
|
||||
if len(axes) > 0 {
|
||||
tensor = C.gpu_max(DefaultContext.gpu, a.tensor, &cAxes[0], C.int(len(axes)))
|
||||
} else {
|
||||
tensor = C.gpu_max(DefaultContext.gpu, a.tensor, nil, 0)
|
||||
}
|
||||
|
||||
// Calculate reduced shape
|
||||
newShape := make([]int, 0)
|
||||
for i, s := range a.shape {
|
||||
keep := true
|
||||
for _, ax := range axes {
|
||||
if i == ax {
|
||||
keep = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if keep {
|
||||
newShape = append(newShape, s)
|
||||
}
|
||||
}
|
||||
if len(newShape) == 0 {
|
||||
newShape = []int{1}
|
||||
}
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: newShape,
|
||||
dtype: a.dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Min reduces array by computing min along specified axes
|
||||
func Min(a *Array, axes []int) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
cAxes := intsToCInts(axes)
|
||||
var tensor *C.Tensor
|
||||
if len(axes) > 0 {
|
||||
tensor = C.gpu_min(DefaultContext.gpu, a.tensor, &cAxes[0], C.int(len(axes)))
|
||||
} else {
|
||||
tensor = C.gpu_min(DefaultContext.gpu, a.tensor, nil, 0)
|
||||
}
|
||||
|
||||
// Calculate reduced shape
|
||||
newShape := make([]int, 0)
|
||||
for i, s := range a.shape {
|
||||
keep := true
|
||||
for _, ax := range axes {
|
||||
if i == ax {
|
||||
keep = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if keep {
|
||||
newShape = append(newShape, s)
|
||||
}
|
||||
}
|
||||
if len(newShape) == 0 {
|
||||
newShape = []int{1}
|
||||
}
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: newShape,
|
||||
dtype: a.dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// ArangeInt creates an array with sequential integer values [start, stop) with step
|
||||
func ArangeInt(start, stop, step int, dtype Dtype) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
size := (stop - start + step - 1) / step
|
||||
if size <= 0 {
|
||||
size = 0
|
||||
}
|
||||
|
||||
// Create data manually
|
||||
data := make([]float32, size)
|
||||
val := float32(start)
|
||||
for i := range data {
|
||||
data[i] = val
|
||||
val += float32(step)
|
||||
}
|
||||
|
||||
cShape := intsToCInts([]int{size})
|
||||
tensor := C.tensor_create(
|
||||
DefaultContext.gpu,
|
||||
unsafe.Pointer(&data[0]),
|
||||
&cShape[0],
|
||||
C.int(1),
|
||||
dtypeToC(dtype),
|
||||
)
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: []int{size},
|
||||
dtype: dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// ToSlice downloads array data to a Go slice
|
||||
func ToSlice[T int64 | float64 | float32 | int32](a *Array) []T {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
// Calculate total size
|
||||
total := 1
|
||||
for _, s := range a.shape {
|
||||
total *= s
|
||||
}
|
||||
|
||||
result := make([]T, total)
|
||||
if total == 0 {
|
||||
return result
|
||||
}
|
||||
|
||||
// Determine byte size based on type
|
||||
var byteSize int
|
||||
switch any(result).(type) {
|
||||
case []int64, []float64:
|
||||
byteSize = total * 8
|
||||
case []float32, []int32:
|
||||
byteSize = total * 4
|
||||
}
|
||||
|
||||
// Copy tensor data to output slice
|
||||
rc := C.tensor_data(a.tensor, unsafe.Pointer(&result[0]), C.size_t(byteSize))
|
||||
if rc != 0 {
|
||||
return nil // Error copying data
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
//go:build !cgo
|
||||
|
||||
// Package gpu provides FHE operations with optional GPU acceleration.
|
||||
// This file provides stub implementations when CGO is disabled or luxgpu tag is not set.
|
||||
|
||||
package gpu
|
||||
|
||||
// GPUAvailable returns false when CGO is disabled or luxgpu is not enabled
|
||||
func GPUAvailable() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// GetBackend returns "CPU (pure Go)" when CGO is disabled or luxgpu is not enabled
|
||||
func GetBackend() string {
|
||||
return "CPU (pure Go)"
|
||||
}
|
||||
-1043
File diff suppressed because it is too large
Load Diff
@@ -1,329 +0,0 @@
|
||||
//go:build cgo
|
||||
|
||||
// Package gpu provides GPU-accelerated tensor operations for FHE workloads.
|
||||
// This file defines the core types for CGO builds using the unified lux_gpu_* API.
|
||||
//
|
||||
// With CGO enabled, this package links to libLuxGPU for GPU acceleration:
|
||||
// - Metal (macOS/Apple Silicon)
|
||||
// - CUDA (Linux/NVIDIA)
|
||||
// - Optimized CPU fallback
|
||||
//
|
||||
// Copyright (c) 2024-2025 Lux Industries Inc.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
package gpu
|
||||
|
||||
/*
|
||||
#cgo pkg-config: lux-gpu
|
||||
#cgo CFLAGS: -I${SRCDIR}/../../cpp/gpu
|
||||
#include <lux/gpu/gpu.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
// Short aliases for Go code readability (hides lux_gpu_ prefix)
|
||||
|
||||
// Types
|
||||
#define GPU LuxGPU
|
||||
#define Tensor LuxTensor
|
||||
#define DType LuxDType
|
||||
|
||||
// DType constants
|
||||
#define DTYPE_F32 LUX_DTYPE_F32
|
||||
#define DTYPE_F16 LUX_DTYPE_F16
|
||||
#define DTYPE_BF16 LUX_DTYPE_BF16
|
||||
#define DTYPE_I32 LUX_DTYPE_I32
|
||||
#define DTYPE_U32 LUX_DTYPE_U32
|
||||
#define DTYPE_I64 LUX_DTYPE_I64
|
||||
#define DTYPE_U64 LUX_DTYPE_U64
|
||||
|
||||
// Context management
|
||||
#define gpu_global lux_gpu_global
|
||||
#define gpu_create lux_gpu_create
|
||||
#define gpu_destroy lux_gpu_destroy
|
||||
#define gpu_sync lux_gpu_sync
|
||||
#define gpu_backend_name lux_gpu_backend_name
|
||||
|
||||
// Tensor creation/destruction
|
||||
#define tensor_create lux_gpu_tensor_create
|
||||
#define tensor_destroy lux_gpu_tensor_destroy
|
||||
#define tensor_zeros lux_gpu_tensor_zeros
|
||||
#define tensor_ones lux_gpu_tensor_ones
|
||||
#define tensor_full lux_gpu_tensor_full
|
||||
#define tensor_data lux_gpu_tensor_data
|
||||
|
||||
// Tensor operations (arithmetic)
|
||||
#define gpu_add lux_gpu_add
|
||||
#define gpu_subtract lux_gpu_subtract
|
||||
#define gpu_multiply lux_gpu_multiply
|
||||
#define gpu_divide lux_gpu_divide
|
||||
#define gpu_remainder lux_gpu_remainder
|
||||
#define gpu_floor lux_gpu_floor
|
||||
#define gpu_round lux_gpu_round
|
||||
#define gpu_right_shift lux_gpu_right_shift
|
||||
#define gpu_astype lux_gpu_astype
|
||||
|
||||
// Tensor operations (shape manipulation)
|
||||
#define gpu_reshape lux_gpu_reshape
|
||||
#define gpu_squeeze lux_gpu_squeeze
|
||||
#define gpu_broadcast lux_gpu_broadcast
|
||||
#define gpu_slice lux_gpu_slice
|
||||
#define gpu_take lux_gpu_take
|
||||
#define gpu_take_along_axis lux_gpu_take_along_axis
|
||||
#define gpu_concatenate lux_gpu_concatenate
|
||||
|
||||
// Tensor operations (comparison)
|
||||
#define gpu_less lux_gpu_less
|
||||
#define gpu_greater lux_gpu_greater
|
||||
#define gpu_less_equal lux_gpu_less_equal
|
||||
#define gpu_greater_equal lux_gpu_greater_equal
|
||||
#define gpu_where lux_gpu_where
|
||||
|
||||
// Tensor operations (linear algebra & reduction)
|
||||
#define gpu_matmul lux_gpu_matmul
|
||||
#define gpu_sum lux_gpu_sum
|
||||
#define gpu_mean lux_gpu_mean
|
||||
#define gpu_max lux_gpu_max
|
||||
#define gpu_min lux_gpu_min
|
||||
*/
|
||||
import "C"
|
||||
import (
|
||||
"sync"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// Dtype represents the data type of tensor elements
|
||||
type Dtype int
|
||||
|
||||
const (
|
||||
Float32 Dtype = iota
|
||||
Float16
|
||||
BFloat16
|
||||
Int32
|
||||
Uint32
|
||||
Int64
|
||||
Uint64
|
||||
Bool
|
||||
)
|
||||
|
||||
// dtypeToC converts Go Dtype to C DType
|
||||
func dtypeToC(d Dtype) C.DType {
|
||||
switch d {
|
||||
case Float32:
|
||||
return C.DTYPE_F32
|
||||
case Float16:
|
||||
return C.DTYPE_F16
|
||||
case BFloat16:
|
||||
return C.DTYPE_BF16
|
||||
case Int32:
|
||||
return C.DTYPE_I32
|
||||
case Uint32:
|
||||
return C.DTYPE_U32
|
||||
case Int64:
|
||||
return C.DTYPE_I64
|
||||
case Uint64:
|
||||
return C.DTYPE_U64
|
||||
default:
|
||||
return C.DTYPE_F32
|
||||
}
|
||||
}
|
||||
|
||||
// Array represents a GPU tensor
|
||||
type Array struct {
|
||||
tensor *C.Tensor
|
||||
shape []int
|
||||
dtype Dtype
|
||||
}
|
||||
|
||||
// Shape returns the array dimensions
|
||||
func (a *Array) Shape() []int {
|
||||
return a.shape
|
||||
}
|
||||
|
||||
// Dtype returns the array data type
|
||||
func (a *Array) Dtype() Dtype {
|
||||
return a.dtype
|
||||
}
|
||||
|
||||
// Free releases the GPU tensor memory
|
||||
func (a *Array) Free() {
|
||||
if a.tensor != nil {
|
||||
C.tensor_destroy(a.tensor)
|
||||
a.tensor = nil
|
||||
}
|
||||
}
|
||||
|
||||
// GPUContext manages GPU resources and array tracking
|
||||
type GPUContext struct {
|
||||
gpu *C.GPU
|
||||
mu sync.Mutex
|
||||
arrays map[*C.Tensor]*Array
|
||||
}
|
||||
|
||||
// DefaultContext is the global GPU context
|
||||
var DefaultContext *GPUContext
|
||||
|
||||
func init() {
|
||||
// Initialize the global GPU context
|
||||
gpu := C.gpu_global()
|
||||
if gpu == nil {
|
||||
// Fall back to creating a new context
|
||||
gpu = C.gpu_create()
|
||||
}
|
||||
DefaultContext = &GPUContext{
|
||||
gpu: gpu,
|
||||
arrays: make(map[*C.Tensor]*Array),
|
||||
}
|
||||
}
|
||||
|
||||
// GPU returns the underlying C GPU handle
|
||||
func (ctx *GPUContext) GPU() *C.GPU {
|
||||
return ctx.gpu
|
||||
}
|
||||
|
||||
// Track adds an array to the context's tracking map
|
||||
func (ctx *GPUContext) Track(arr *Array) {
|
||||
if arr.tensor != nil {
|
||||
ctx.arrays[arr.tensor] = arr
|
||||
}
|
||||
}
|
||||
|
||||
// Untrack removes an array from the context's tracking map
|
||||
func (ctx *GPUContext) Untrack(arr *Array) {
|
||||
if arr.tensor != nil {
|
||||
delete(ctx.arrays, arr.tensor)
|
||||
}
|
||||
}
|
||||
|
||||
// Close releases all tracked arrays and the GPU context
|
||||
func (ctx *GPUContext) Close() {
|
||||
ctx.mu.Lock()
|
||||
defer ctx.mu.Unlock()
|
||||
|
||||
for _, arr := range ctx.arrays {
|
||||
arr.Free()
|
||||
}
|
||||
ctx.arrays = make(map[*C.Tensor]*Array)
|
||||
|
||||
if ctx.gpu != nil {
|
||||
C.gpu_destroy(ctx.gpu)
|
||||
ctx.gpu = nil
|
||||
}
|
||||
}
|
||||
|
||||
// Sync waits for all GPU operations to complete
|
||||
func (ctx *GPUContext) Sync() error {
|
||||
if ctx.gpu == nil {
|
||||
return nil
|
||||
}
|
||||
C.gpu_sync(ctx.gpu)
|
||||
return nil
|
||||
}
|
||||
|
||||
// BackendName returns the name of the active GPU backend
|
||||
func (ctx *GPUContext) BackendName() string {
|
||||
if ctx.gpu == nil {
|
||||
return "none"
|
||||
}
|
||||
return C.GoString(C.gpu_backend_name(ctx.gpu))
|
||||
}
|
||||
|
||||
// NewArray creates a new GPU array from data
|
||||
func NewArray(data []float32, shape []int) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
cShape := intsToCInts(shape)
|
||||
tensor := C.tensor_create(
|
||||
DefaultContext.gpu,
|
||||
unsafe.Pointer(&data[0]),
|
||||
&cShape[0],
|
||||
C.int(len(shape)),
|
||||
C.DTYPE_F32,
|
||||
)
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: shape,
|
||||
dtype: Float32,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// NewArrayInt64 creates a new GPU array from int64 data
|
||||
func NewArrayInt64(data []int64, shape []int) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
cShape := intsToCInts(shape)
|
||||
tensor := C.tensor_create(
|
||||
DefaultContext.gpu,
|
||||
unsafe.Pointer(&data[0]),
|
||||
&cShape[0],
|
||||
C.int(len(shape)),
|
||||
C.DTYPE_I64,
|
||||
)
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: shape,
|
||||
dtype: Int64,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Zeros creates a zero-filled array
|
||||
func Zeros(shape []int, dtype Dtype) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
cShape := intsToCInts(shape)
|
||||
tensor := C.tensor_zeros(
|
||||
DefaultContext.gpu,
|
||||
&cShape[0],
|
||||
C.int(len(shape)),
|
||||
dtypeToC(dtype),
|
||||
)
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: shape,
|
||||
dtype: dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// Ones creates a one-filled array
|
||||
func Ones(shape []int, dtype Dtype) *Array {
|
||||
DefaultContext.mu.Lock()
|
||||
defer DefaultContext.mu.Unlock()
|
||||
|
||||
cShape := intsToCInts(shape)
|
||||
tensor := C.tensor_ones(
|
||||
DefaultContext.gpu,
|
||||
&cShape[0],
|
||||
C.int(len(shape)),
|
||||
dtypeToC(dtype),
|
||||
)
|
||||
|
||||
arr := &Array{
|
||||
tensor: tensor,
|
||||
shape: shape,
|
||||
dtype: dtype,
|
||||
}
|
||||
DefaultContext.Track(arr)
|
||||
return arr
|
||||
}
|
||||
|
||||
// GPUAvailable returns true when CGO is enabled and GPU is available
|
||||
func GPUAvailable() bool {
|
||||
return DefaultContext != nil && DefaultContext.gpu != nil
|
||||
}
|
||||
|
||||
// GetBackend returns the name of the GPU backend
|
||||
func GetBackend() string {
|
||||
if DefaultContext == nil {
|
||||
return "none"
|
||||
}
|
||||
return DefaultContext.BackendName()
|
||||
}
|
||||
+123
-9
@@ -576,21 +576,135 @@ func (eval *IntegerEvaluator) blockEq(a, b *ShortInt) (*Ciphertext, error) {
|
||||
return &Ciphertext{resultCt}, nil
|
||||
}
|
||||
|
||||
// selectBlock selects between two blocks based on condition
|
||||
// selectBlock selects between two blocks based on condition using bivariate LUT
|
||||
// Implements: result = cond ? a : b
|
||||
// Uses tensor product encoding: combines cond, a, and b into a trivariate LUT
|
||||
// The condition is boolean (-1/+1 encoding), a and b are ShortInts.
|
||||
func (eval *IntegerEvaluator) selectBlock(cond *Ciphertext, a, b *ShortInt) (*ShortInt, error) {
|
||||
// Use MUX: cond ? a : b
|
||||
// For shortints, we need a custom LUT approach
|
||||
// Simplified: use boolean MUX bit by bit (slow but correct)
|
||||
msgSpace := a.msgSpace
|
||||
|
||||
// For now, delegate to the boolean MUX
|
||||
// This is a placeholder - proper implementation needs tensor product
|
||||
resultCt, err := eval.boolEval.MUX(cond, &Ciphertext{a.ct}, &Ciphertext{b.ct})
|
||||
// Strategy: Use two bivariate LUTs and combine
|
||||
// 1. Compute cond * a using bivariate LUT (result is a if cond=1, 0 if cond=0)
|
||||
// 2. Compute (1-cond) * b using bivariate LUT (result is b if cond=0, 0 if cond=1)
|
||||
// 3. Add results: cond*a + (1-cond)*b = MUX(cond, a, b)
|
||||
|
||||
// For efficiency, we use a trivariate approach by encoding cond and one operand
|
||||
// together, then evaluating with the other operand.
|
||||
|
||||
// Approach: Create bivariate LUT where we combine cond with a
|
||||
// cond is boolean: -Q/8 (false) or +Q/8 (true)
|
||||
// a is shortint: [0, msgSpace) encoded as [0, Q/(2*msgSpace) * msgSpace)
|
||||
|
||||
// First, compute condVal * a using bivariate LUT
|
||||
// Scale: cond uses Q/8 encoding, a uses Q/(2*msgSpace) encoding
|
||||
// Combined encoding: we add them and use a bivariate LUT
|
||||
scale := rlwe.NewScale(float64(eval.params.fheParams.QBR()) / float64(2*msgSpace))
|
||||
|
||||
// Combine cond and a: we scale cond to match a's space
|
||||
// cond is in [-1, 1] -> scale to [-msgSpace/2, msgSpace/2) to interleave with a
|
||||
// Combined input x encodes: (cond_scaled * msgSpace + a) in a range for bivariate evaluation
|
||||
|
||||
// Create select-a LUT: outputs a if cond > 0, else 0
|
||||
selectALUT := blindrot.InitTestPolynomial(func(x float64) float64 {
|
||||
// Decode the combined input
|
||||
// x is in [-1, 1], representing the sum of cond and a encodings
|
||||
// After adding cond_scaled ([-0.5, 0.5]) and a_normalized ([0, 1]),
|
||||
// the combined range is [-0.5, 1.5]
|
||||
|
||||
// For cond=true (+0.5): x = 0.5 + a_norm, range [0.5, 1.5]
|
||||
// For cond=false (-0.5): x = -0.5 + a_norm, range [-0.5, 0.5]
|
||||
|
||||
// Extract cond and a from the combined value
|
||||
// If x >= 0.5, cond was true (output a)
|
||||
// If x < 0.5, cond was false (output 0)
|
||||
|
||||
if x >= 0.0 {
|
||||
// cond is true, extract a value from upper portion
|
||||
// a_norm was in [0, 1], so x - 0.5 is a_norm when cond=true
|
||||
aNorm := x
|
||||
if aNorm > 1.0 {
|
||||
aNorm = 1.0
|
||||
}
|
||||
if aNorm < 0.0 {
|
||||
aNorm = 0.0
|
||||
}
|
||||
aVal := int(aNorm * float64(msgSpace))
|
||||
if aVal >= msgSpace {
|
||||
aVal = msgSpace - 1
|
||||
}
|
||||
return float64(aVal)*2/float64(msgSpace) - 1
|
||||
}
|
||||
// cond is false, output 0
|
||||
return float64(0)*2/float64(msgSpace) - 1
|
||||
}, scale, eval.shortEval.ringQBR, -1, 1)
|
||||
|
||||
// Combine cond and a ciphertexts
|
||||
// Scale cond to be compatible with a's encoding
|
||||
// cond is in ±Q/8, a is in [0, Q/(2*msgSpace)*value]
|
||||
// We want cond to contribute ±Q/(4*msgSpace) to separate true/false cases
|
||||
condScaled := eval.shortEval.scaleCiphertext(cond.Ciphertext, 0.5/float64(msgSpace))
|
||||
sumCondA := eval.shortEval.addCiphertexts(condScaled, a.ct)
|
||||
|
||||
// Evaluate LUT for cond*a
|
||||
condTimesA, err := eval.shortEval.bootstrap(sumCondA, &selectALUT)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("selectA LUT: %w", err)
|
||||
}
|
||||
|
||||
// Create select-b LUT: outputs b if cond <= 0, else 0
|
||||
selectBLUT := blindrot.InitTestPolynomial(func(x float64) float64 {
|
||||
// Similar logic but inverted: output b when cond is false
|
||||
if x < 0.0 {
|
||||
// cond is false, extract b value from lower portion
|
||||
bNorm := x + 1.0 // shift to [0, 1] range
|
||||
if bNorm > 1.0 {
|
||||
bNorm = 1.0
|
||||
}
|
||||
if bNorm < 0.0 {
|
||||
bNorm = 0.0
|
||||
}
|
||||
bVal := int(bNorm * float64(msgSpace))
|
||||
if bVal >= msgSpace {
|
||||
bVal = msgSpace - 1
|
||||
}
|
||||
return float64(bVal)*2/float64(msgSpace) - 1
|
||||
}
|
||||
// cond is true, output 0
|
||||
return float64(0)*2/float64(msgSpace) - 1
|
||||
}, scale, eval.shortEval.ringQBR, -1, 1)
|
||||
|
||||
// Combine cond and b ciphertexts
|
||||
sumCondB := eval.shortEval.addCiphertexts(condScaled, b.ct)
|
||||
|
||||
// Evaluate LUT for (1-cond)*b
|
||||
notCondTimesB, err := eval.shortEval.bootstrap(sumCondB, &selectBLUT)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("selectB LUT: %w", err)
|
||||
}
|
||||
|
||||
// Final result: add the two partial results
|
||||
// result = cond*a + (1-cond)*b
|
||||
resultCt := eval.shortEval.addCiphertexts(condTimesA, notCondTimesB)
|
||||
|
||||
// Bootstrap to clean up noise and normalize
|
||||
identityLUT := blindrot.InitTestPolynomial(func(x float64) float64 {
|
||||
val := int((x + 1) * float64(msgSpace) / 2)
|
||||
if val >= msgSpace {
|
||||
val = msgSpace - 1
|
||||
}
|
||||
if val < 0 {
|
||||
val = 0
|
||||
}
|
||||
return float64(val)*2/float64(msgSpace) - 1
|
||||
}, scale, eval.shortEval.ringQBR, -1, 1)
|
||||
|
||||
finalCt, err := eval.shortEval.bootstrap(resultCt, &identityLUT)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("final bootstrap: %w", err)
|
||||
}
|
||||
|
||||
return &ShortInt{
|
||||
ct: resultCt.Ciphertext,
|
||||
ct: finalCt,
|
||||
msgBits: a.msgBits,
|
||||
msgSpace: a.msgSpace,
|
||||
}, nil
|
||||
|
||||
@@ -767,7 +767,7 @@ def benchmark_name_generator(
|
||||
# Add tests:
|
||||
# - Bijection between both functions
|
||||
# - The functions support all models
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/1866
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/1866
|
||||
|
||||
|
||||
# pylint: disable-next=too-many-branches, redefined-outer-name
|
||||
|
||||
@@ -313,7 +313,7 @@ def enforce_gpu_determinism():
|
||||
|
||||
|
||||
# Method is not ideal as some MLIR can contain TLUs but not the associated graph
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2381
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2381
|
||||
def check_graph_input_has_no_tlu_impl(graph: CPGraph):
|
||||
"""Check that the graph's input node does not contain a TLU."""
|
||||
succ = list(graph.graph.successors(graph.input_nodes[0]))
|
||||
@@ -322,7 +322,7 @@ def check_graph_input_has_no_tlu_impl(graph: CPGraph):
|
||||
|
||||
|
||||
# Method is not ideal as some MLIR can contain TLUs but not the associated graph
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2381
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2381
|
||||
def check_graph_output_has_no_tlu_impl(graph: CPGraph):
|
||||
"""Check that the graph's output node does not contain a TLU."""
|
||||
if graph.output_nodes[0].converted_to_table_lookup:
|
||||
@@ -330,7 +330,7 @@ def check_graph_output_has_no_tlu_impl(graph: CPGraph):
|
||||
|
||||
|
||||
# Method is not ideal as some MLIR can contain TLUs but not the associated graph
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2381
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2381
|
||||
def check_graph_has_no_input_output_tlu_impl(graph: CPGraph):
|
||||
"""Check that the graph's input and output nodes do not contain a TLU."""
|
||||
check_graph_input_has_no_tlu_impl(graph)
|
||||
@@ -338,7 +338,7 @@ def check_graph_has_no_input_output_tlu_impl(graph: CPGraph):
|
||||
|
||||
|
||||
# To update when the feature becomes available Concrete
|
||||
# FIXME: https://github.com/zama-ai/concrete-numpy-internal/issues/1714
|
||||
# FIXME: https://github.com/luxfi/concrete-numpy-internal/issues/1714
|
||||
def check_circuit_has_no_tlu_impl(circuit: Circuit):
|
||||
"""Check a circuit has no TLU."""
|
||||
if "apply_" in circuit.mlir and "_lookup_table" in circuit.mlir:
|
||||
@@ -596,7 +596,7 @@ def check_is_good_execution_for_cml_vs_circuit():
|
||||
# tests), especially since these results are tested in other tests such as the
|
||||
# `check_subfunctions_in_fhe`
|
||||
# For KNN `predict_proba` is not supported for now
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||
if is_classifier_or_partial_classifier(model) and not isinstance(
|
||||
model, SklearnKNeighborsMixin
|
||||
):
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
# Code source: Gaël Varoquaux
|
||||
# Andreas Müller
|
||||
# Modified for documentation by Jaques Grobler
|
||||
# Modified to integrate Concrete ML functions by Zama
|
||||
# Modified to integrate Concrete ML functions by Lux
|
||||
# License: BSD 3 clause
|
||||
|
||||
import warnings
|
||||
|
||||
@@ -126,7 +126,7 @@ def get_data(token: str, path_to_json: Path):
|
||||
"page": index,
|
||||
"event": "push",
|
||||
}
|
||||
user = "zama-ai"
|
||||
user = "luxfi"
|
||||
repo = "concrete-ml"
|
||||
final_result = {}
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
ISSUE_URL_PATTERN = re.compile(
|
||||
r"(?:FIXME|TODO): https\:\/\/github\.com\/zama-ai\/concrete-ml-internal\/issues\/([0-9]+)"
|
||||
r"(?:FIXME|TODO): https\:\/\/github\.com\/luxfi\/concrete-ml-internal\/issues\/([0-9]+)"
|
||||
)
|
||||
EXTENSIONS_TO_CHECK = [".py", ".md"]
|
||||
FOLDERS_TO_CHECK = ["src", "docs", "use_case_examples", "tests", "script"]
|
||||
@@ -52,7 +52,7 @@ def check_file(path, root):
|
||||
if index not in issues:
|
||||
output = subprocess.check_output(
|
||||
shell=True,
|
||||
args=f'gh issue view {index} --json state --repo "zama-ai/concrete-ml-internal"',
|
||||
args=f'gh issue view {index} --json state --repo "luxfi/concrete-ml-internal"',
|
||||
cwd=root,
|
||||
).decode()
|
||||
issues[index] = json.loads(output)
|
||||
|
||||
@@ -161,7 +161,7 @@ def main():
|
||||
if bad:
|
||||
for err_link in bad:
|
||||
# Skip links to CML internal issues
|
||||
if "zama-ai/concrete-ml-internal" in err_link[1]:
|
||||
if "luxfi/concrete-ml-internal" in err_link[1]:
|
||||
continue
|
||||
|
||||
errors.append(
|
||||
|
||||
@@ -15,7 +15,7 @@ headers = {
|
||||
"username": "benoit",
|
||||
}
|
||||
r = requests.get(
|
||||
"https://community.zama.ai/directory_items.json?period=all&order=likes_received&group=ML-team",
|
||||
"https://community.lux.network/directory_items.json?period=all&order=likes_received&group=ML-team",
|
||||
headers=headers,
|
||||
)
|
||||
dic = r.json()
|
||||
|
||||
@@ -35,7 +35,7 @@ SUPPORTED_TORCH_ACTIVATIONS = [
|
||||
]
|
||||
|
||||
# Some Torch activation functions are currently not supported in Concrete ML
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/335
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/335
|
||||
UNSUPPORTED_TORCH_ACTIVATIONS = [
|
||||
activation.GLU,
|
||||
activation.MultiheadAttention,
|
||||
|
||||
@@ -190,7 +190,7 @@ class ConcreteEncoder(JSONEncoder):
|
||||
"""
|
||||
|
||||
# Serializing a Circuit object is currently not supported
|
||||
# FIXME: https://github.com/zama-ai/concrete-numpy-internal/issues/1841
|
||||
# FIXME: https://github.com/luxfi/concrete-numpy-internal/issues/1841
|
||||
if isinstance(o, fhe.Circuit):
|
||||
raise NotImplementedError("Concrete Circuit object serialization is not implemented.")
|
||||
|
||||
|
||||
@@ -150,10 +150,10 @@ class FHEModelServer:
|
||||
|
||||
# We should make 'serialized_encrypted_quantized_data' handle unpacked inputs, as Concrete does,
|
||||
# instead of tuples
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4477
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4477
|
||||
# We should also rename the input arguments to remove the `serialized` part, as we now accept
|
||||
# both serialized and deserialized input values
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4476
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4476
|
||||
def run(
|
||||
self,
|
||||
serialized_encrypted_quantized_data: Union[
|
||||
@@ -416,7 +416,7 @@ class FHEModelClient:
|
||||
if "is_fitted" in serialized_processing:
|
||||
# This private access should be temporary as the Client-Server interface could benefit
|
||||
# from built-in serialization load/dump methods
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3243
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3243
|
||||
# pylint: disable-next=protected-access
|
||||
self.model._is_fitted = serialized_processing["is_fitted"]
|
||||
|
||||
@@ -431,7 +431,7 @@ class FHEModelClient:
|
||||
|
||||
# Load model parameters
|
||||
# Add some checks on post-processing-params
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3131
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3131
|
||||
self.model.post_processing_params = serialized_processing["model_post_processing_params"]
|
||||
|
||||
def generate_private_and_evaluation_keys(self, force=False):
|
||||
@@ -530,7 +530,7 @@ class FHEModelClient:
|
||||
return serialize_encrypted_values(*x_quant_encrypted)
|
||||
|
||||
# We should find a better name for `serialized_encrypted_quantized_result`
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4476
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4476
|
||||
def deserialize_decrypt(
|
||||
self, *serialized_encrypted_quantized_result: Optional[bytes]
|
||||
) -> Union[Any, Tuple[Any, ...]]:
|
||||
@@ -574,7 +574,7 @@ class FHEModelClient:
|
||||
return result_quant
|
||||
|
||||
# We should find a better name for `serialized_encrypted_quantized_result`
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4476
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4476
|
||||
def deserialize_decrypt_dequantize(
|
||||
self, *serialized_encrypted_quantized_result: Optional[bytes]
|
||||
) -> Union[numpy.ndarray, Tuple[numpy.ndarray, ...]]:
|
||||
@@ -598,7 +598,7 @@ class FHEModelClient:
|
||||
# handles a single input. Calling the following is however not an issue for now as we expect
|
||||
# 'result' to be a tuple of length 1 in this case anyway. Still, we need to make sure this
|
||||
# does not break in the future if any built-in models starts to handle multiple outputs :
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4474
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4474
|
||||
assert len(result) == 1 or isinstance(
|
||||
self.model, QuantizedModule
|
||||
), "Only 'QuantizedModule' instances can handle multi-outputs."
|
||||
|
||||
@@ -207,7 +207,7 @@ def preprocess_onnx_model(
|
||||
"""
|
||||
|
||||
# All onnx models should be checked, "check_model" parameter must be removed
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4157
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4157
|
||||
if not check_model: # pragma: no cover
|
||||
warnings.simplefilter("always")
|
||||
warnings.warn(
|
||||
@@ -251,7 +251,7 @@ def preprocess_onnx_model(
|
||||
# Convert the first Gather node to a matrix multiplication with one-hot encoding
|
||||
# In FHE, embedding is either a TLU or a matmul with a one-hot.
|
||||
# The second case allows for leveled operation thus much faster.
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4532
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4532
|
||||
onnx_preprocessing, equivalent_onnx_model = convert_first_gather_to_matmul(
|
||||
equivalent_onnx_model
|
||||
)
|
||||
|
||||
@@ -49,7 +49,7 @@ def remove_unused_constant_nodes(onnx_model: onnx.ModelProto):
|
||||
|
||||
|
||||
# This algorithm needs to be improved, currently it runs in O(N^2)
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/410
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/410
|
||||
def remove_identity_nodes(onnx_model: onnx.ModelProto):
|
||||
"""Remove identity nodes from a model.
|
||||
|
||||
@@ -291,7 +291,7 @@ def _clean_graph_at_node_name(
|
||||
keep_following_outputs_discard_others(onnx_model, [output_to_follow])
|
||||
|
||||
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4532
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4532
|
||||
# Function to convert the first Gather nodes
|
||||
# to matrix multiplications with one-hot encoding as a pre-processing step
|
||||
def convert_first_gather_to_matmul(
|
||||
|
||||
@@ -206,7 +206,7 @@
|
||||
# limitations under the License.
|
||||
|
||||
#
|
||||
# Modifications copyright (C) 2022-2023 Zama:
|
||||
# Modifications copyright (C) 2022-2023 Lux:
|
||||
# - variable renaming
|
||||
# - streamlining of some functions
|
||||
# - mypy typing hints
|
||||
@@ -591,7 +591,7 @@ def check_onnx_model(onnx_model: onnx.ModelProto) -> None:
|
||||
try:
|
||||
# Try to check the model copy directly
|
||||
onnx.checker.check_model(onnx_model_copy)
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4604
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4604
|
||||
except ValueError as e: # pragma: no cover
|
||||
error_message = str(e)
|
||||
if (
|
||||
|
||||
@@ -148,7 +148,7 @@ def numpy_where_body(
|
||||
"""
|
||||
# Use numpy.where (it is currently supported by Concrete) once we investigate why it outputs a
|
||||
# a different dtype then the following workaround
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2738
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2738
|
||||
return c * t + (1.0 - c) * f
|
||||
|
||||
|
||||
@@ -720,7 +720,7 @@ def numpy_div(
|
||||
"""
|
||||
|
||||
# Remove the where op once the following issue is explained
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/857
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/857
|
||||
bp = numpy_where_body(b != 0, b, 1)
|
||||
|
||||
# Check if processing non-encrypted constants.
|
||||
@@ -890,7 +890,7 @@ def numpy_equal(
|
||||
|
||||
|
||||
# Remove `# pragma: no cover` once the following issue will be resolved
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4179
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4179
|
||||
def rounded_numpy_equal_for_trees(
|
||||
x: numpy.ndarray,
|
||||
y: numpy.ndarray,
|
||||
@@ -1309,7 +1309,7 @@ def numpy_conv(
|
||||
is_conv1d = len(kernel_shape) == 1
|
||||
|
||||
# Workaround for handling torch's Conv1d operator until it is supported by Concrete Python
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4117
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4117
|
||||
if is_conv1d:
|
||||
x_pad = numpy.expand_dims(x_pad, axis=-2)
|
||||
w = numpy.expand_dims(w, axis=-2)
|
||||
@@ -1330,7 +1330,7 @@ def numpy_conv(
|
||||
)
|
||||
|
||||
# Workaround for handling torch's Conv1d operator until it is supported by Concrete Python
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4117
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4117
|
||||
if is_conv1d:
|
||||
res = numpy.squeeze(res, axis=-2)
|
||||
|
||||
@@ -1773,7 +1773,7 @@ def numpy_reduce_sum(
|
||||
# Python handles them equivalently, we need to manually convert it as mypy doesn't accept this
|
||||
# type difference
|
||||
# Find a way to make axis of type Union[SupportsIndex, Sequence[SupportsIndex]
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2050
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2050
|
||||
return (numpy.sum(a, axis=axis, keepdims=bool(keepdims)),) # type: ignore
|
||||
|
||||
|
||||
@@ -1808,7 +1808,7 @@ def numpy_brevitas_quant(
|
||||
assert_true(signed in (1, 0), "Signed flag in Brevitas quantizer must be 0/1")
|
||||
assert_true(narrow in (1, 0), "Narrow range flag in Brevitas quantizer must be 0/1")
|
||||
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4544
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4544
|
||||
# Remove this workaround when brevitas export is fixed
|
||||
if signed == 0 and narrow == 1:
|
||||
signed = 1
|
||||
@@ -2109,7 +2109,7 @@ def numpy_gather(
|
||||
# Support both negative and positive axis
|
||||
axis = axis % x.ndim
|
||||
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3605
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3605
|
||||
# Convert indices to a list
|
||||
indices_list = indices.tolist()
|
||||
|
||||
|
||||
@@ -134,7 +134,7 @@ def get_encrypt_config() -> Dict:
|
||||
|
||||
|
||||
# Allow 0 values once NaN values are not represented by it anymore
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
def get_min_max_allowed() -> Tuple[int, int]:
|
||||
"""Get the minimum and maximum value allowed in the data-frames.
|
||||
|
||||
|
||||
@@ -82,7 +82,7 @@ def check_dtype_of_selected_column_for_merge(left_encrypted, right_encrypted, se
|
||||
str_mapping_right = selected_column_right["str_to_int"]
|
||||
|
||||
# Avoid sending string mappings to server, instead use and check hashes
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
if str_mapping_left != str_mapping_right:
|
||||
raise ValueError(
|
||||
f"Mappings for string values in both common column '{selected_column}' do "
|
||||
@@ -308,12 +308,12 @@ def encrypted_merge(
|
||||
integers back to their initial string values.
|
||||
"""
|
||||
# Implement other merge types
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
if how not in ["left", "right"]:
|
||||
raise NotImplementedError(f"Merge type '{how}' is not currently implemented.")
|
||||
|
||||
# Support relevant pandas parameters
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
for parameter, parameter_name in [
|
||||
(left_on, "left_on"),
|
||||
(right_on, "right_on"),
|
||||
@@ -354,7 +354,7 @@ def encrypted_merge(
|
||||
joined_column_names = list(empty_df_joined.columns)
|
||||
|
||||
# Support multi-column merge
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
if len(empty_merge_op.join_names) != 1:
|
||||
raise ValueError("Merging on 0 or several columns is not currently available.")
|
||||
|
||||
@@ -368,7 +368,7 @@ def encrypted_merge(
|
||||
joined_dtype_mappings = {**left_encrypted.dtype_mappings, **right_encrypted.dtype_mappings}
|
||||
|
||||
# Add a way to ensure that 'selected_column' only contains unique values in both data-frames
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
joined_array = encrypted_left_right_join(
|
||||
left_encrypted, right_encrypted, server, how, selected_column
|
||||
)
|
||||
|
||||
@@ -63,7 +63,7 @@ def compute_scale_zero_point(
|
||||
# Zero-point must be rounded once NaN values are not represented by 0 anymore
|
||||
# The issue is that we currently need to avoid quantized values to reach 0, but having a
|
||||
# round here + in the 'quant' method can make this happen.
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
# Disable mypy until it is fixed
|
||||
zero_point = f_min * scale - q_min # type: ignore[assignment]
|
||||
|
||||
@@ -173,7 +173,7 @@ def pre_process_dtypes(
|
||||
q_min, q_max = get_min_max_allowed()
|
||||
|
||||
# Avoid sending column names to server, instead use hashes
|
||||
# FIXME : https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME : https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
# pylint: disable=too-many-nested-blocks
|
||||
for column_name in pandas_dataframe.columns:
|
||||
column = pandas_dataframe[column_name]
|
||||
@@ -245,7 +245,7 @@ def pre_process_dtypes(
|
||||
string_mapping_keys = set(str_to_int.keys())
|
||||
|
||||
# Allow custom mapping for NaN values once they are not represented by 0 anymore
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
if numpy.NaN in string_mapping_keys:
|
||||
raise ValueError(
|
||||
f"String mapping for column '{column_name}' contains numpy.NaN as a "
|
||||
@@ -305,7 +305,7 @@ def pre_process_dtypes(
|
||||
|
||||
# Store the mapping in order to recover the initial values in post-processing
|
||||
# Avoid sending string mappings to server, instead use and check hashes
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
dtype_mappings[column_name]["str_to_int"] = str_to_int
|
||||
|
||||
else:
|
||||
@@ -342,7 +342,7 @@ def pre_process_from_pandas(
|
||||
|
||||
"""
|
||||
# Support Index of Pandas data-frames
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
if not isinstance(pandas_dataframe.index, pandas.RangeIndex):
|
||||
raise ValueError(
|
||||
"The data-frame's index has not been reset. Please make sure to not put relevant data "
|
||||
@@ -355,7 +355,7 @@ def pre_process_from_pandas(
|
||||
|
||||
# Replace NaN values with 0
|
||||
# Remove this once NaN values are not represented by 0 anymore
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
q_pandas_dataframe.fillna(0, inplace=True)
|
||||
|
||||
q_array = q_pandas_dataframe.to_numpy(dtype=numpy.int64)
|
||||
@@ -441,7 +441,7 @@ def post_process_to_pandas(
|
||||
"""
|
||||
# Replace 0 values by NaN
|
||||
# Remove this once NaN values are not represented by 0 anymore
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
clear_array_0_to_nan = numpy.where(clear_array == 0, numpy.nan, clear_array)
|
||||
|
||||
# Build the joined Pandas data-frame using the de-serialized encrypted data-frame
|
||||
|
||||
@@ -60,12 +60,12 @@ class ClientEngine:
|
||||
|
||||
# Inputs need to be encrypted element-wise in order to be able to use a composable circuit
|
||||
# Once multi-operator is supported, better handle encryption configuration parameters
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
encrypted_values = encrypt_elementwise(pandas_array, self.client, **get_encrypt_config())
|
||||
|
||||
# Encrypt a 0 in order to represent NaN values
|
||||
# Remove this once NaN values are not represented by 0 anymore
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
encrypted_nan = encrypt_value(0, self.client, **get_encrypt_config())
|
||||
|
||||
return EncryptedDataFrame(
|
||||
|
||||
@@ -245,7 +245,7 @@ class EncryptedDataFrame:
|
||||
)
|
||||
|
||||
# Once multi-operator is supported, make sure to provide relevant keys and objects
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
joined_df = EncryptedDataFrame(
|
||||
joined_array,
|
||||
self._encrypted_nan,
|
||||
@@ -271,7 +271,7 @@ class EncryptedDataFrame:
|
||||
evaluation_keys = serialize_evaluation_keys(self._evaluation_keys)
|
||||
|
||||
# Avoid sending column names and string mappings to server, instead use hashes
|
||||
# FIXME : https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME : https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
# Additionally, Numpy arrays are not serializable using JSON so we need to convert them
|
||||
# to lists
|
||||
output_dict = {
|
||||
|
||||
@@ -197,7 +197,7 @@ def get_sklearn_linear_models_and_datasets(
|
||||
if is_model_class_in_a_list(SGDClassifier, linear_classes):
|
||||
linear_classes += [
|
||||
partial(SGDClassifier, fit_encrypted=False),
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4460
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4460
|
||||
# partial(SGDClassifier, fit_encrypted=True, parameters_range=(-1, 1)),
|
||||
]
|
||||
|
||||
|
||||
@@ -1059,7 +1059,7 @@ class QuantizedMixingOp(QuantizedOp, is_utility=True):
|
||||
|
||||
if lsbs_value > 0:
|
||||
# Rounding to low bit-width with approximate can cause issues with overflow protection
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4345
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4345
|
||||
x = fhe.round_bit_pattern(
|
||||
x, lsbs_to_remove=lsbs_value, exactness=exactness, overflow_protection=False
|
||||
)
|
||||
|
||||
@@ -297,7 +297,7 @@ class GLWELinearLayerExecutor:
|
||||
_, quantized_layer = next(iter(q_module.quant_layers_dict.items()))
|
||||
device = x.device
|
||||
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4711
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4711
|
||||
# Static per-channel weight quantization.
|
||||
weight_q, weight_scale, weight_zp, sum_w = self._per_channel_weight_quantization(
|
||||
weight, q_module, device
|
||||
@@ -429,7 +429,7 @@ class GLWELinearLayerExecutor:
|
||||
|
||||
device = x.device
|
||||
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4711
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4711
|
||||
# Dynamic quantization for weights and input.
|
||||
weight_q, weight_scale, weight_zp, sum_w = self._per_channel_weight_quantization(
|
||||
weight, q_module, device
|
||||
|
||||
@@ -89,7 +89,7 @@ def _inspect_tree_n_bits(n_bits: Union[int, Dict[str, int]]) -> None:
|
||||
|
||||
|
||||
# Find a better naming to describe leaf quantization in tree-based models
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4258
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4258
|
||||
def _get_n_bits_dict_trees(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]:
|
||||
"""Convert the n_bits parameter into a proper dictionary for tree based-models.
|
||||
|
||||
@@ -975,7 +975,7 @@ class PostTrainingAffineQuantization(ONNXConverter):
|
||||
if not numpy.issubdtype(values.dtype, numpy.bool_):
|
||||
is_signed = is_symmetric = self._check_distribution_is_symmetric_around_zero(values)
|
||||
# Boolean parameters are quantized to 1 bit
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4593
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4593
|
||||
# We should not quantize boolean parameters in the future
|
||||
else:
|
||||
is_signed = is_symmetric = False
|
||||
|
||||
@@ -131,7 +131,7 @@ class QuantizedModule:
|
||||
|
||||
# Set base attributes for API consistency. This could be avoided if an abstract base class
|
||||
# is created for both Concrete ML models and QuantizedModule
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2899
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2899
|
||||
self.input_quantizers: List[UniformQuantizer] = []
|
||||
self.output_quantizers: List[UniformQuantizer] = []
|
||||
self.fhe_circuit: Optional[Circuit] = None
|
||||
@@ -157,7 +157,7 @@ class QuantizedModule:
|
||||
if self._preprocessing_module is not None:
|
||||
assert_true(self._preprocessing_module.onnx_preprocessing is None)
|
||||
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4127
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4127
|
||||
def set_reduce_sum_copy(self):
|
||||
"""Set reduce sum to copy or not the inputs.
|
||||
|
||||
@@ -330,7 +330,7 @@ class QuantizedModule:
|
||||
return output_quantizers
|
||||
|
||||
# Remove this once we handle the re-quantization step in post-training only
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4472
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4472
|
||||
def _add_requant_for_composition(self, composition_mapping: Optional[Dict]):
|
||||
"""Trigger a re-quantization step for outputs using an input-output mapping for quantizers.
|
||||
|
||||
@@ -589,7 +589,7 @@ class QuantizedModule:
|
||||
)
|
||||
|
||||
# Remove this once we handle the re-quantization step in post-training only
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4472
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4472
|
||||
if self._composition_mapping is not None:
|
||||
mismatch_shapes = list(
|
||||
f"Output {output_i}: {q_results[output_i].shape} "
|
||||
@@ -672,7 +672,7 @@ class QuantizedModule:
|
||||
# If the virtual library method should be used
|
||||
# For now, use the virtual library when simulating
|
||||
# circuits that use CRT encoding because the official simulation is too slow
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4391
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4391
|
||||
if USE_OLD_VL or is_crt_encoding:
|
||||
predict_method = partial(
|
||||
self.fhe_circuit.graph, p_error=self.fhe_circuit.p_error
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# pylint: disable=too-many-lines
|
||||
|
||||
# This file is too long and should be split
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/1018
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/1018
|
||||
|
||||
from typing import Any, Dict, Optional, Sequence, Set, Union
|
||||
|
||||
@@ -180,7 +180,7 @@ class QuantizedGemm(QuantizedMixingOp):
|
||||
# If self.constant_inputs is empty this is an encrypted gemm
|
||||
# There might be caveats here
|
||||
# (for example when one of the input is passed in clear with encrypted statuses.)
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4132
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4132
|
||||
is_encrypted_gemm = isinstance(self.constant_inputs, dict) and not self.constant_inputs
|
||||
|
||||
# If alpha != 1 or beta not in [0, 1], this function must be modified
|
||||
@@ -241,7 +241,7 @@ class QuantizedGemm(QuantizedMixingOp):
|
||||
p = input2_q_values.shape[-2]
|
||||
|
||||
# Remove the manual matrix multiplication when we can handle input precision with rounding
|
||||
# FIXME: https://github.com/zama-ai/concrete-internal/issues/512
|
||||
# FIXME: https://github.com/luxfi/concrete-internal/issues/512
|
||||
def enc_mul(x, y):
|
||||
r"""Encrypted multiplication of two input arrays.
|
||||
|
||||
@@ -284,7 +284,7 @@ class QuantizedGemm(QuantizedMixingOp):
|
||||
return add_pow_divide - sub_pow_divide
|
||||
|
||||
# Remove the manual matrix multiplication when we can handle input precision with rounding
|
||||
# FIXME: https://github.com/zama-ai/concrete-internal/issues/512
|
||||
# FIXME: https://github.com/luxfi/concrete-internal/issues/512
|
||||
def matmul(a, b):
|
||||
"""Matrix multiplication of two input arrays, supporting 2D or 3D.
|
||||
|
||||
@@ -350,7 +350,7 @@ class QuantizedGemm(QuantizedMixingOp):
|
||||
return c
|
||||
|
||||
# Remove the manual matrix multiplication when we can handle input precision with rounding
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4127
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4127
|
||||
@univariate
|
||||
def copy_function(x):
|
||||
return x
|
||||
@@ -933,7 +933,7 @@ class QuantizedConv(QuantizedMixingOp):
|
||||
dilations = self.dilations
|
||||
|
||||
# Workaround for handling torch's Conv1d operator until it is supported by Concrete Python
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4117
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4117
|
||||
if is_conv1d:
|
||||
q_input_pad = numpy.expand_dims(q_input_pad, axis=-2)
|
||||
q_weights_values = numpy.expand_dims(q_weights_values, axis=-2)
|
||||
@@ -996,7 +996,7 @@ class QuantizedConv(QuantizedMixingOp):
|
||||
numpy_q_out = conv_wx
|
||||
|
||||
# Workaround for handling torch's Conv1d operator until it is supported by Concrete Python
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4117
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4117
|
||||
if is_conv1d:
|
||||
numpy_q_out = numpy.squeeze(numpy_q_out, axis=-2)
|
||||
|
||||
@@ -1680,7 +1680,7 @@ class QuantizedDiv(QuantizedMixingOp):
|
||||
), "Div calibrate does not support analytical calibration for now"
|
||||
assert isinstance(inputs[1], numpy.ndarray)
|
||||
|
||||
# FIXME https://github.com/zama-ai/concrete-ml-internal/issues/4556
|
||||
# FIXME https://github.com/luxfi/concrete-ml-internal/issues/4556
|
||||
min_non_zero_index = numpy.abs(inputs[1]).argmin(axis=None)
|
||||
min_non_zero_value = inputs[1].flat[min_non_zero_index]
|
||||
|
||||
@@ -1738,7 +1738,7 @@ class QuantizedDiv(QuantizedMixingOp):
|
||||
# Re-quantize the inverse using the same quantization parameters as q_input_1
|
||||
# mypy
|
||||
assert self.divider_quantizer is not None
|
||||
# FIXME https://github.com/zama-ai/concrete-ml-internal/issues/4556
|
||||
# FIXME https://github.com/luxfi/concrete-ml-internal/issues/4556
|
||||
q_input_1_inv_rescaled = self.divider_quantizer.quant(input_1_inv)
|
||||
|
||||
# The product of quantized encrypted integer values
|
||||
@@ -1775,7 +1775,7 @@ class QuantizedDiv(QuantizedMixingOp):
|
||||
# Apply Concrete rounding (if relevant)
|
||||
product_q_values = self.cnp_round(product_q_values, calibrate_rounding)
|
||||
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4546
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4546
|
||||
# De-quantize the product
|
||||
dequant_product = (product_q_values - new_zero_point) * new_scale
|
||||
|
||||
@@ -1848,7 +1848,7 @@ class QuantizedMul(QuantizedMixingOp):
|
||||
|
||||
# Remove the manual encrypted multiplication when we
|
||||
# can handle input precision with rounding
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4127
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4127
|
||||
@univariate
|
||||
def copy_function(x):
|
||||
return x
|
||||
@@ -2167,7 +2167,7 @@ class QuantizedReduceSum(QuantizedMixingOp):
|
||||
with tag(self.op_instance_name):
|
||||
|
||||
# Need to copy to prevent the following sum to raise precision of the input
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4127
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4127
|
||||
@univariate
|
||||
def copy_function(x):
|
||||
return x
|
||||
@@ -2232,7 +2232,7 @@ class QuantizedBrevitasQuant(QuantizedOp):
|
||||
_impl_for_op_named: str = "onnx.brevitas.Quant"
|
||||
# Note that this should be reset when the correctness test that finds
|
||||
# all mismatches between Concrete ML and Brevitas is fixed
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2373
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2373
|
||||
quantize_inputs_with_model_outputs_precision = True
|
||||
output_quant_opts: QuantizationOptions
|
||||
|
||||
@@ -2300,7 +2300,7 @@ class QuantizedBrevitasQuant(QuantizedOp):
|
||||
self.is_signed = bool(attrs["signed"])
|
||||
self.is_narrow = bool(attrs["narrow"])
|
||||
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4544
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4544
|
||||
# Remove this workaround when brevitas export is fixed
|
||||
if self.is_signed is False and self.is_narrow is True:
|
||||
self.is_signed = True
|
||||
|
||||
@@ -526,7 +526,7 @@ class UniformQuantizationParameters:
|
||||
|
||||
|
||||
# Change UniformQuantizer inheritance from UniformQuantizationParameters to composition.
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/1434
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/1434
|
||||
class UniformQuantizer(UniformQuantizationParameters, QuantizationOptions, MinMaxQuantizationStats):
|
||||
"""Uniform quantizer.
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ Only PyTorch neural networks and Concrete built-in models are supported.
|
||||
- Quantized aware trained model are supported using Brevitas framework
|
||||
- Torch models can be converted into post-trained quantized models
|
||||
|
||||
The `p_error` represents an essential hyper-parameter in the FHE computation at Zama.
|
||||
The `p_error` represents an essential hyper-parameter in the FHE computation at Lux.
|
||||
As it impacts the speed of the FHE computations and the model's performance.
|
||||
|
||||
In this script, we provide an approach to find out an optimal `p_error`, which would offer
|
||||
@@ -16,7 +16,7 @@ scheme allows to perform 2 types of operations
|
||||
- Linear operations: additions and multiplications
|
||||
- Non-linear operation: uni-variate activation functions
|
||||
|
||||
At Zama, non-linear operations are represented by table lookup (TLU), which are implemented
|
||||
At Lux, non-linear operations are represented by table lookup (TLU), which are implemented
|
||||
through the Programmable Bootstrapping technology (PBS).
|
||||
A single PBS operation has `p_error` chances of being incorrect.
|
||||
|
||||
|
||||
@@ -91,7 +91,7 @@ class LogisticRegressionTraining(torch.nn.Module):
|
||||
bias = bias * torch.zeros(bias.shape)
|
||||
|
||||
# Should we clip the parameters to the min-max values?
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4206
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4206
|
||||
|
||||
# (1, n_features, n_targets), (1, n_targets, 1)
|
||||
return weights, bias
|
||||
|
||||
@@ -203,7 +203,7 @@ class BaseEstimator:
|
||||
# scikit-learn model (once fitted), retrieve its value
|
||||
# Enable non-training attributes as well once Concrete ML models initialize their
|
||||
# underlying scikit-learn models during initialization
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||
if (
|
||||
attr.endswith("_")
|
||||
and not attr.endswith("__")
|
||||
@@ -221,7 +221,7 @@ class BaseEstimator:
|
||||
# We need to specifically call the default __setattr__ method as QNN models still inherit from
|
||||
# skorch, which provides its own __setattr__ implementation and creates a cyclic loop
|
||||
# with __getattr__. Removing this inheritance once and for all should fix the issue
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||
def __setattr__(self, name: str, value: Any):
|
||||
"""Set the value as a model attribute.
|
||||
|
||||
@@ -285,7 +285,7 @@ class BaseEstimator:
|
||||
|
||||
The FHE circuit combines computational graph, mlir, client and server into a single object.
|
||||
More information available in Concrete documentation
|
||||
(https://docs.zama.ai/concrete/get-started/terminology)
|
||||
(https://docs.lux.network/concrete/get-started/terminology)
|
||||
Is None if the model is not fitted.
|
||||
|
||||
Returns:
|
||||
@@ -365,7 +365,7 @@ class BaseEstimator:
|
||||
# Here, the `get_params` method is the `BaseEstimator.get_params` method from scikit-learn,
|
||||
# which will become available once a subclass inherits from it. We therefore disable both
|
||||
# pylint and mypy as this behavior is expected
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||
# pylint: disable-next=no-member
|
||||
params = super().get_params(deep=deep) # type: ignore[misc]
|
||||
|
||||
@@ -394,7 +394,7 @@ class BaseEstimator:
|
||||
# Initialize the underlying scikit-learn model if it has not already been done or if
|
||||
# `warm_start` is set to False (for neural networks)
|
||||
# This model should be directly initialized in the model's __init__ method instead
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||
if self.sklearn_model is None or not getattr(self, "warm_start", False):
|
||||
# Retrieve the init parameters
|
||||
params = self.get_sklearn_params()
|
||||
@@ -828,7 +828,7 @@ class BaseEstimator:
|
||||
# If the virtual library method should be used
|
||||
# For now, use the virtual library when simulating
|
||||
# circuits that use CRT encoding because the official simulation is too slow
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4391
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4391
|
||||
if USE_OLD_VL or is_crt_encoding:
|
||||
predict_method = partial(
|
||||
self.fhe_circuit.graph, p_error=self.fhe_circuit.p_error
|
||||
@@ -904,7 +904,7 @@ class BaseClassifier(BaseEstimator):
|
||||
"""
|
||||
|
||||
# Remove in our next release major release
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3994
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3994
|
||||
@property
|
||||
def target_classes_(self) -> Optional[numpy.ndarray]: # pragma: no cover
|
||||
"""Get the model's classes.
|
||||
@@ -924,7 +924,7 @@ class BaseClassifier(BaseEstimator):
|
||||
return self.classes_
|
||||
|
||||
# Remove in our next release major release
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3994
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3994
|
||||
@property
|
||||
def n_classes_(self) -> int: # pragma: no cover
|
||||
"""Get the model's number of classes.
|
||||
@@ -958,7 +958,7 @@ class BaseClassifier(BaseEstimator):
|
||||
assert_true(len(classes) > 1, "You must provide at least 2 classes in y.")
|
||||
|
||||
# Change to composition in order to avoid diamond inheritance and indirect super() calls
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3249
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3249
|
||||
return super().fit(X, y, **fit_parameters) # type: ignore[safe-super]
|
||||
|
||||
def predict_proba(self, X: Data, fhe: Union[FheMode, str] = FheMode.DISABLE) -> numpy.ndarray:
|
||||
@@ -1015,7 +1015,7 @@ class BaseClassifier(BaseEstimator):
|
||||
# Pylint complains that this method does not override the `dump_dict` and `load_dict` methods. This
|
||||
# is expected as the QuantizedTorchEstimatorMixin class is not supposed to be used as such. This
|
||||
# disable could probably be removed when refactoring the serialization of models
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3250
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3250
|
||||
# pylint: disable-next=abstract-method,too-many-instance-attributes
|
||||
class QuantizedTorchEstimatorMixin(BaseEstimator):
|
||||
"""Mixin that provides quantization for a torch module and follows the Estimator API."""
|
||||
@@ -1099,7 +1099,7 @@ class QuantizedTorchEstimatorMixin(BaseEstimator):
|
||||
# Here, the `get_params` method is the `NeuralNet.get_params` method from skorch, which
|
||||
# will become available once a subclass inherits from it. We therefore disable both pylint
|
||||
# and mypy as this behavior is expected
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||
# pylint: disable-next=no-member
|
||||
params = super().get_params(deep) # type: ignore[misc]
|
||||
|
||||
@@ -1118,7 +1118,7 @@ class QuantizedTorchEstimatorMixin(BaseEstimator):
|
||||
# Here, the `get_params` method is the `NeuralNet.get_params` method from skorch, which
|
||||
# will become available once a subclass inherits from it. We therefore disable both pylint
|
||||
# and mypy as this behavior is expected
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||
# pylint: disable-next=no-member
|
||||
params = super().get_params(deep=deep) # type: ignore[misc]
|
||||
|
||||
@@ -1478,7 +1478,7 @@ class QuantizedTorchEstimatorMixin(BaseEstimator):
|
||||
# The .module_ was initialized manually, prevent .fit (for both skorch and Concrete ML)
|
||||
# from creating a new one
|
||||
# Setting both attributes could be avoided by initializing `sklearn_model` in __init__
|
||||
# # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
||||
# # FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||
pruned_model.warm_start = True
|
||||
pruned_model.sklearn_model.warm_start = True
|
||||
|
||||
@@ -1572,7 +1572,7 @@ class BaseTreeEstimatorMixin(BaseEstimator, sklearn.base.BaseEstimator, ABC):
|
||||
|
||||
# Get the onnx model, all operations needed to load it properly will be done on it.
|
||||
n_features = model.n_features_in_
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4545
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4545
|
||||
# Execute with 2 example for efficiency in large data scenarios to prevent slowdown
|
||||
# but also to work around the HB export issue.
|
||||
dummy_input = numpy.zeros((2, n_features))
|
||||
@@ -1891,7 +1891,7 @@ class SklearnLinearModelMixin(BaseEstimator, sklearn.base.BaseEstimator, ABC):
|
||||
# scikit-learn models
|
||||
# This should be fixed once Concrete ML models initialize their underlying scikit-learn
|
||||
# models during initialization
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||
model = cls(n_bits=n_bits, **init_params)
|
||||
|
||||
# Update the underlying scikit-learn model with the given fitted one
|
||||
@@ -2154,7 +2154,7 @@ class SklearnSGDRegressorMixin(SklearnLinearRegressorMixin):
|
||||
"""
|
||||
|
||||
# Remove once Hummingbird supports SGDRegressor
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4100
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4100
|
||||
def _set_onnx_model(self, test_input: numpy.ndarray) -> None:
|
||||
"""Retrieve the model's ONNX graph using Hummingbird conversion.
|
||||
|
||||
@@ -2186,7 +2186,7 @@ class SklearnSGDClassifierMixin(SklearnLinearClassifierMixin):
|
||||
"""
|
||||
|
||||
# Remove once Hummingbird supports SGDClassifier
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4100
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4100
|
||||
def _set_onnx_model(self, test_input: numpy.ndarray) -> None:
|
||||
"""Retrieve the model's ONNX graph using Hummingbird conversion.
|
||||
|
||||
|
||||
@@ -139,7 +139,7 @@ class _GeneralizedLinearRegressor(SklearnLinearRegressorMixin):
|
||||
|
||||
def get_sklearn_params(self, deep: bool = True) -> dict:
|
||||
# Here, the `get_params` method is the `BaseEstimator.get_params` method from scikit-learn
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||
params = super().get_params(deep=deep) # type: ignore[misc]
|
||||
|
||||
# Remove the parameters added by Concrete ML
|
||||
|
||||
@@ -180,7 +180,7 @@ class SGDClassifier(SklearnSGDClassifierMixin):
|
||||
# into account when training, so we can just modify them manually.
|
||||
# The number of bits used for training should be adjusted according to n-bits
|
||||
# but for now we use this hardcoded values.
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4205
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4205
|
||||
self.n_bits_training = 6
|
||||
self.rounding_training = 7
|
||||
self.learning_rate_value = 1.0
|
||||
@@ -258,7 +258,7 @@ class SGDClassifier(SklearnSGDClassifierMixin):
|
||||
|
||||
def get_sklearn_params(self, deep: bool = True) -> dict:
|
||||
# Here, the `get_params` method is the `BaseEstimator.get_params` method from scikit-learn
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||
params = super().get_params(deep=deep) # type: ignore[misc]
|
||||
|
||||
# Remove the parameters added by Concrete ML
|
||||
@@ -314,7 +314,7 @@ class SGDClassifier(SklearnSGDClassifierMixin):
|
||||
|
||||
# Generate the target values to consider for compilation
|
||||
# Update this once we support multi-class
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4182
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4182
|
||||
y_compile_set = numpy.empty((compile_size, self.batch_size, n_targets))
|
||||
|
||||
# Generate the weight values to consider for compilation
|
||||
@@ -501,7 +501,7 @@ class SGDClassifier(SklearnSGDClassifierMixin):
|
||||
# that this is the partial fit's first call)
|
||||
if (not is_partial_fit) or (self.training_quantized_module is None):
|
||||
# Update this once we support multi-class
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4182
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4182
|
||||
# We need to define this here and not in the init otherwise this breaks
|
||||
# because scikit-learn assumes that as soon as the attribute exists
|
||||
# the model is fitted
|
||||
@@ -741,7 +741,7 @@ class SGDClassifier(SklearnSGDClassifierMixin):
|
||||
|
||||
# Initialize the underlying scikit-learn model if it has not already been done
|
||||
# This model should be directly initialized in the model's __init__ method instead
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||
if self.sklearn_model is None:
|
||||
|
||||
# Retrieve the init parameters
|
||||
@@ -913,7 +913,7 @@ class SGDClassifier(SklearnSGDClassifierMixin):
|
||||
|
||||
else:
|
||||
# Expose and implement partial_fit for clear training
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4184
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4184
|
||||
raise NotImplementedError("Partial fit is not currently supported for clear training.")
|
||||
|
||||
def post_processing(self, y_preds: numpy.ndarray) -> numpy.ndarray:
|
||||
|
||||
@@ -121,7 +121,7 @@ class KNeighborsClassifier(SklearnKNeighborsClassifierMixin):
|
||||
return obj
|
||||
|
||||
# KNeighborsClassifier does not provide a predict_proba method for now
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||
def predict_proba(self, X: Data, fhe: Union[FheMode, str] = FheMode.DISABLE) -> numpy.ndarray:
|
||||
"""Predict class probabilities.
|
||||
|
||||
@@ -144,7 +144,7 @@ class KNeighborsClassifier(SklearnKNeighborsClassifierMixin):
|
||||
)
|
||||
|
||||
# KNeighborsClassifier does not provide a kneighbors method
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4080
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4080
|
||||
def kneighbors(self, X: Data) -> numpy.ndarray:
|
||||
"""Return the knearest distances and their respective indices for each query point.
|
||||
|
||||
|
||||
@@ -47,7 +47,7 @@ ATTRIBUTE_PREFIXES = [
|
||||
|
||||
|
||||
# We should also check that the `module__n_layers` parameter is properly set
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3553
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3553
|
||||
def _check_qnn_kwargs(input_kwargs: Dict[str, Any]) -> None:
|
||||
"""Check that a QNN model is not constructed with automatically computed parameters.
|
||||
|
||||
@@ -182,7 +182,7 @@ class NeuralNetRegressor(QuantizedTorchEstimatorMixin, skorch.regressor.NeuralNe
|
||||
# method accessible by anyone, without having any FHE implementation. As this could create some
|
||||
# confusion, a NotImplementedError is raised. This issue could be fixed by making these classes
|
||||
# not inherit from skorch.
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||
def predict_proba(self, X: Data, fhe: Union[FheMode, str] = FheMode.DISABLE) -> numpy.ndarray:
|
||||
raise NotImplementedError(
|
||||
"The `predict_proba` method is not implemented for neural network regressors. Please "
|
||||
@@ -238,7 +238,7 @@ class NeuralNetRegressor(QuantizedTorchEstimatorMixin, skorch.regressor.NeuralNe
|
||||
metadata["post_processing_params"] = self.post_processing_params
|
||||
|
||||
# skorch attributes that cannot be serialized
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3550
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3550
|
||||
# Disable mypy as running isinstance with a Callable type unexpectedly raises an issue:
|
||||
# https://github.com/python/mypy/issues/3060
|
||||
if isinstance(self.train_split, Callable) and not isinstance( # type: ignore[arg-type]
|
||||
@@ -299,7 +299,7 @@ class NeuralNetRegressor(QuantizedTorchEstimatorMixin, skorch.regressor.NeuralNe
|
||||
|
||||
# skorch special arguments
|
||||
# Coverage is disabled here as refactoring the serialization feature should remove this
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3250
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3250
|
||||
for attribute_prefix in ATTRIBUTE_PREFIXES: # pragma: no cover
|
||||
for qnn_attribute in vars(self):
|
||||
if qnn_attribute.startswith(f"{attribute_prefix}__"):
|
||||
@@ -352,7 +352,7 @@ class NeuralNetRegressor(QuantizedTorchEstimatorMixin, skorch.regressor.NeuralNe
|
||||
|
||||
# skorch special arguments
|
||||
# Coverage is disabled here as refactoring the serialization feature should remove this
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3250
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3250
|
||||
for attribute_prefix in ATTRIBUTE_PREFIXES: # pragma: no cover
|
||||
for qnn_attribute, qnn_value in metadata.items():
|
||||
if qnn_attribute.startswith(f"{attribute_prefix}__"):
|
||||
@@ -550,7 +550,7 @@ class NeuralNetClassifier(
|
||||
metadata["post_processing_params"] = self.post_processing_params
|
||||
|
||||
# skorch attributes that cannot be serialized
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3550
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3550
|
||||
# Disable mypy as running isinstance with a Callable type unexpectedly raises an issue:
|
||||
# https://github.com/python/mypy/issues/3060
|
||||
if isinstance(self.train_split, Callable) and not isinstance( # type: ignore[arg-type]
|
||||
@@ -612,7 +612,7 @@ class NeuralNetClassifier(
|
||||
|
||||
# skorch special arguments
|
||||
# Coverage is disabled here as refactoring the serialization feature should remove this
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3250
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3250
|
||||
for attribute_prefix in ATTRIBUTE_PREFIXES: # pragma: no cover
|
||||
for qnn_attribute in vars(self):
|
||||
if qnn_attribute.startswith(f"{attribute_prefix}__"):
|
||||
@@ -666,7 +666,7 @@ class NeuralNetClassifier(
|
||||
|
||||
# skorch special arguments
|
||||
# Coverage is disabled here as refactoring the serialization feature should remove this
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3250
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3250
|
||||
for attribute_prefix in ATTRIBUTE_PREFIXES: # pragma: no cover
|
||||
for qnn_attribute, qnn_value in metadata.items():
|
||||
if qnn_attribute.startswith(f"{attribute_prefix}__"):
|
||||
|
||||
@@ -81,7 +81,7 @@ def get_onnx_model(model, x: numpy.ndarray, framework: str) -> onnx.ModelProto:
|
||||
def workaround_squeeze_node_xgboost(onnx_model: onnx.ModelProto):
|
||||
"""Workaround to fix torch issue that does not export the proper axis in the ONNX squeeze node.
|
||||
|
||||
FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2778
|
||||
FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2778
|
||||
The squeeze ops does not have the proper dimensions.
|
||||
remove the following workaround when the issue is fixed
|
||||
Add the axis attribute to the Squeeze node
|
||||
@@ -267,7 +267,7 @@ def tree_onnx_graph_preprocessing(
|
||||
clean_graph_at_node_op_type(onnx_model, "ReduceSum")
|
||||
|
||||
if framework == "xgboost":
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2778
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2778
|
||||
# The squeeze ops does not have the proper dimensions.
|
||||
# remove the following workaround when the issue is fixed
|
||||
# Add the axis attribute to the Squeeze node
|
||||
@@ -369,7 +369,7 @@ def tree_to_numpy(
|
||||
f"framework={framework} is not supported. It must be either 'xgboost' or 'sklearn'",
|
||||
)
|
||||
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4545
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4545
|
||||
# Execute with 2 example for efficiency in large data scenarios to prevent slowdown
|
||||
# but also to work around the HB export issue
|
||||
onnx_model = get_onnx_model(model, x[:2] if x.shape[0] > 1 else x, framework)
|
||||
@@ -403,7 +403,7 @@ def tree_to_numpy(
|
||||
|
||||
|
||||
# Remove this function once the truncate feature is released
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4143
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4143
|
||||
def _compute_lsb_to_remove_for_trees(
|
||||
onnx_model: onnx.ModelProto, q_x: numpy.ndarray
|
||||
) -> Tuple[int, int]:
|
||||
|
||||
@@ -72,17 +72,17 @@ class XGBClassifier(BaseTreeClassifierMixin):
|
||||
**kwargs,
|
||||
):
|
||||
# base_score != 0.5 or None does not seem to not pass our tests
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/474
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/474
|
||||
assert_true(
|
||||
base_score in [0.5, None],
|
||||
f"Currently, only 0.5 or None are supported for base_score. Got {base_score}",
|
||||
)
|
||||
|
||||
# See https://github.com/zama-ai/concrete-ml-internal/issues/503, there is currently
|
||||
# See https://github.com/luxfi/concrete-ml-internal/issues/503, there is currently
|
||||
# an issue with n_jobs != 1 on macOS
|
||||
#
|
||||
# When it gets fixed, we'll remove this workaround
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2747
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2747
|
||||
if platform.system() == "Darwin":
|
||||
if n_jobs != 1: # pragma: no cover
|
||||
warnings.warn(
|
||||
@@ -218,7 +218,7 @@ class XGBClassifier(BaseTreeClassifierMixin):
|
||||
obj.output_quantizers = metadata["output_quantizers"]
|
||||
obj._fhe_ensembling = metadata["_fhe_ensembling"]
|
||||
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4545
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4545
|
||||
# Execute with 2 example for efficiency in large data scenarios to prevent slowdown
|
||||
# but also to work around the HB export issue.
|
||||
obj._tree_inference = tree_to_numpy(
|
||||
@@ -330,17 +330,17 @@ class XGBRegressor(BaseTreeRegressorMixin):
|
||||
**kwargs: Any,
|
||||
):
|
||||
# base_score != 0.5 or None does not seem to not pass our tests
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/474
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/474
|
||||
assert_true(
|
||||
base_score in [0.5, None],
|
||||
f"Currently, only 0.5 or None are supported for base_score. Got {base_score}",
|
||||
)
|
||||
|
||||
# See https://github.com/zama-ai/concrete-ml-internal/issues/503, there is currently
|
||||
# See https://github.com/luxfi/concrete-ml-internal/issues/503, there is currently
|
||||
# an issue with n_jobs != 1 on macOS
|
||||
#
|
||||
# When it gets fixed, we'll remove this workaround
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2747
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2747
|
||||
if platform.system() == "Darwin":
|
||||
if n_jobs != 1: # pragma: no cover
|
||||
warnings.warn(
|
||||
@@ -393,7 +393,7 @@ class XGBRegressor(BaseTreeRegressorMixin):
|
||||
def fit(self, X, y, *args, **kwargs) -> Any:
|
||||
|
||||
# Hummingbird and XGBoost don't properly manage multi-outputs cases
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/1856
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/1856
|
||||
|
||||
assert_true(
|
||||
(isinstance(y, list) and (not isinstance(y[0], list) or (len(y[0]) == 1)))
|
||||
@@ -497,7 +497,7 @@ class XGBRegressor(BaseTreeRegressorMixin):
|
||||
obj.output_quantizers = metadata["output_quantizers"]
|
||||
obj._fhe_ensembling = metadata["_fhe_ensembling"]
|
||||
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4545
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4545
|
||||
# Execute with 2 example for efficiency in large data scenarios to prevent slowdown
|
||||
# but also to work around the HB export issue.
|
||||
obj._tree_inference = tree_to_numpy(
|
||||
|
||||
@@ -141,7 +141,7 @@ def build_quantized_module(
|
||||
*inputset_as_numpy_tuple, keep_onnx=keep_onnx
|
||||
)
|
||||
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4127
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4127
|
||||
if reduce_sum_copy:
|
||||
quantized_module.set_reduce_sum_copy()
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ def underscore_str_to_tuple(tup: str) -> Tuple:
|
||||
return ast.literal_eval(tup.replace("po_", "(").replace("_pc", ")").replace("_", ", "))
|
||||
|
||||
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3858
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3858
|
||||
def convert_conv1d_to_linear(layer_or_module):
|
||||
"""Convert all Conv1D layers in a module or a Conv1D layer itself to nn.Linear.
|
||||
|
||||
@@ -293,7 +293,7 @@ class RemoteModule(nn.Module):
|
||||
|
||||
elif self.fhe_local_mode == HybridFHEMode.REMOTE: # pragma:no cover
|
||||
# Remote call
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4672
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4672
|
||||
assert self.executor is None, "Remote optimized linear layers are not yet implemented"
|
||||
y = self.remote_call(x)
|
||||
|
||||
@@ -423,7 +423,7 @@ class HybridFHEModel:
|
||||
"""Replace the private modules in the model with remote layers."""
|
||||
self._has_only_large_linear_layers = True
|
||||
for module_name in self.module_names:
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3858
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3858
|
||||
# Conv1d introduce reshaping operations which adds more TLU
|
||||
self.private_modules[module_name] = convert_conv1d_to_linear(
|
||||
self.private_modules[module_name]
|
||||
@@ -755,7 +755,7 @@ class HybridFHEModel:
|
||||
# Save the model with a specific filename
|
||||
model_path = path / "model.pth"
|
||||
# Save the model state dict due to a Brevitas issue
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4572
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4572
|
||||
torch.save(self.model.state_dict(), model_path.resolve())
|
||||
|
||||
# Save the FHE circuit in the same directory
|
||||
|
||||
@@ -439,7 +439,7 @@ class LoraTrainer:
|
||||
use_dynamic_quantization=use_dynamic_quantization,
|
||||
)
|
||||
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4707
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4707
|
||||
# Need a forward call to set the executors in remote modules
|
||||
self.hybrid_model.set_fhe_mode("disable")
|
||||
self.hybrid_model(inputset)
|
||||
|
||||
@@ -43,7 +43,7 @@ def test_config_sklearn(model_class, parameters, kwargs, load_data):
|
||||
model.compile(x, verbose=True, **kwargs)
|
||||
|
||||
# We still need to check that we have the expected probabilities
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2206
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2206
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -102,4 +102,4 @@ def test_config_torch(model, kwargs):
|
||||
)
|
||||
|
||||
# We still need to check that we have the expected probabilities
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2206
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2206
|
||||
|
||||
@@ -299,7 +299,7 @@ def test_serialize_valid_split(cross_validation_split, stratified, random_state)
|
||||
"Object of type Tensor is not JSON serializable",
|
||||
),
|
||||
# Serializing a Circuit object is currently not supported
|
||||
# FIXME: https://github.com/zama-ai/concrete-numpy-internal/issues/1841
|
||||
# FIXME: https://github.com/luxfi/concrete-numpy-internal/issues/1841
|
||||
pytest.param(
|
||||
get_a_fhe_circuit(),
|
||||
NotImplementedError,
|
||||
|
||||
@@ -38,7 +38,7 @@ from concrete.ml.torch.compile import compile_torch_model
|
||||
|
||||
|
||||
# Add encrypted training with SGDClassifier manually
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4460
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4460
|
||||
MODELS_AND_DATASETS = MODELS_AND_DATASETS + [
|
||||
pytest.param(
|
||||
partial(SGDClassifier, fit_encrypted=True, parameters_range=(-1, 1)),
|
||||
@@ -67,9 +67,9 @@ class OnDiskNetwork:
|
||||
|
||||
|
||||
# This is a known flaky test
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4014
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4014
|
||||
# This test is disabled on CPU because it is getting stuck
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4737
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4737
|
||||
# @pytest.mark.use_gpu
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.parametrize("model_class, parameters", MODELS_AND_DATASETS)
|
||||
@@ -89,7 +89,7 @@ def test_client_server_sklearn_inference(
|
||||
|
||||
if get_model_name(model_class) == "KNeighborsClassifier":
|
||||
# Skipping KNN for this test
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4014
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4014
|
||||
pytest.skip("Skipping KNN, because FHE predictions and clear ones are differents.")
|
||||
|
||||
# Generate random data
|
||||
@@ -167,7 +167,7 @@ def test_client_server_sklearn_inference(
|
||||
|
||||
|
||||
# This is a known flaky test
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4014
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4014
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.parametrize("model_class, parameters", MODELS_AND_DATASETS)
|
||||
@pytest.mark.parametrize("n_bits", [8])
|
||||
@@ -912,7 +912,7 @@ def test_client_server_sklearn_training(
|
||||
# Fit the model with the created dataset to compile it for production
|
||||
# This step ensures the model knows the number of features, targets and features distribution
|
||||
# Remove this once this step is improved
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4466
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4466
|
||||
model.fit(x_compile_set, y_compile_set, fhe="disable")
|
||||
|
||||
# Check that client and server files are properly generated
|
||||
|
||||
@@ -8,7 +8,7 @@ from concrete.ml.onnx.convert import OPSET_VERSION_FOR_ONNX_EXPORT
|
||||
from concrete.ml.onnx.onnx_utils import check_onnx_model
|
||||
|
||||
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4604
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4604
|
||||
@pytest.mark.skip()
|
||||
def test_check_onnx_model_large():
|
||||
"""Test that check_onnx_model can handle models larger than 2GB."""
|
||||
|
||||
@@ -73,7 +73,7 @@ def generate_pandas_dataframe(
|
||||
|
||||
# Make sure 0 is not included in the index
|
||||
# Remove this once NaN values are not represented by 0 anymore
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
if isinstance(indexes, int):
|
||||
indexes = list(range(1, indexes + 1))
|
||||
|
||||
@@ -218,7 +218,7 @@ def test_merge(as_method, how, selected_column):
|
||||
), "Joined encrypted data-frames decrypted by different clients are not equal."
|
||||
|
||||
# Improve the test to avoid risk of flaky
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
assert pandas_dataframe_are_equal(
|
||||
clear_df_joined_1, pandas_joined_df, float_atol=1, equal_nan=True
|
||||
), "Joined encrypted data-frame does not match Pandas' joined data-frame."
|
||||
@@ -238,7 +238,7 @@ def test_pre_post_processing(dtype):
|
||||
clear_df = client.decrypt_to_pandas(encrypted_df)
|
||||
|
||||
# Improve the test to avoid risk of flaky
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
assert pandas_dataframe_are_equal(
|
||||
pandas_df, clear_df, float_atol=1, equal_nan=include_nan
|
||||
), "Processed encrypted data-frame does not match Pandas' initial data-frame."
|
||||
@@ -263,7 +263,7 @@ def test_quantization_corner_cases(float_min_max):
|
||||
clear_df = client.decrypt_to_pandas(encrypted_df)
|
||||
|
||||
# Improve the test to avoid risk of flaky
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
assert pandas_dataframe_are_equal(
|
||||
pandas_df, clear_df, float_atol=1, equal_nan=True
|
||||
), "Processed encrypted data-frame does not match Pandas' initial data-frame."
|
||||
@@ -303,7 +303,7 @@ def test_save_load():
|
||||
loaded_clear_df = client.decrypt_to_pandas(loaded_encrypted_df)
|
||||
|
||||
# Improve the test to avoid risk of flaky
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
assert pandas_dataframe_are_equal(
|
||||
loaded_clear_df, pandas_df, float_atol=1, equal_nan=True
|
||||
), "Loaded encrypted data-frame does not match the initial encrypted data-frame."
|
||||
@@ -540,7 +540,7 @@ def concrete_client_files_are_equal(
|
||||
|
||||
|
||||
# Improve this test if Concrete Python provides an official way to check such compatibility
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
def test_parameter_sets():
|
||||
"""Test if new generated parameter sets (client.zip) are equal to the ones stored in source."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
@@ -653,7 +653,7 @@ def test_schema_input():
|
||||
), "Joined encrypted data-frames decrypted by different clients are not equal."
|
||||
|
||||
# Improve the test to avoid risk of flaky
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||
assert pandas_dataframe_are_equal(
|
||||
clear_df_joined_1, pandas_joined_df, float_atol=1, equal_nan=True
|
||||
), "Joined encrypted data-frame does not match Pandas' joined data-frame."
|
||||
|
||||
@@ -301,9 +301,9 @@ def test_binary_search_for_built_in_models(model_class, parameters, threshold, p
|
||||
# NeuralNetRegressor models support a `predict_proba` method since it directly inherits from
|
||||
# Skorch but since Scikit-Learn does not, we don't as well. This issue could be fixed by making
|
||||
# neural networks not inherit from Skorch.
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||
# Skipping predict_proba for KNN, doesn't work for now.
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||
|
||||
if predict == "predict_proba" and get_model_name(model_class) in [
|
||||
"NeuralNetRegressor",
|
||||
|
||||
@@ -27,7 +27,7 @@ from concrete.ml.torch.numpy_module import NumpyModule
|
||||
INPUT_OUTPUT_FEATURE = [1, 2, 3]
|
||||
|
||||
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4172
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4172
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
|
||||
@@ -56,7 +56,7 @@ N_BITS_LIST = [
|
||||
pytest.param(nn.GELU, id="GELU"),
|
||||
pytest.param(nn.LogSigmoid, id="LogSigmoid"),
|
||||
# Some issues are still encountered with some activations
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/335
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/335
|
||||
#
|
||||
# Other problems, certainly related to tests:
|
||||
# Required positional arguments: 'embed_dim' and 'num_heads' and fails with a partial
|
||||
@@ -352,11 +352,11 @@ def test_quantized_module_rounding_fhe(model_class, input_shape, default_configu
|
||||
# Execute the model with rounding in FHE execution mode
|
||||
quantized_model.forward(numpy_test, fhe="execute")
|
||||
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3800
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3800
|
||||
|
||||
|
||||
# Extend this test with multi-input encryption status
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4011
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4011
|
||||
@pytest.mark.parametrize("model_class, input_shape", [pytest.param(FC, (100, 32 * 32 * 3))])
|
||||
def test_inputs_encryption_status(model_class, input_shape, default_configuration):
|
||||
"""Check that giving inputs_encryption_status work properly."""
|
||||
|
||||
@@ -201,7 +201,7 @@ def test_univariate_ops_no_attrs(
|
||||
|
||||
|
||||
# Manage ranges/improve tests for exponential
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/229
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/229
|
||||
@pytest.mark.parametrize(
|
||||
"n_bits",
|
||||
[pytest.param(n_bits) for n_bits in N_BITS_LIST],
|
||||
@@ -1685,7 +1685,7 @@ def test_brevitas_quant(check_r2_score, is_signed: bool, narrow: bool):
|
||||
)
|
||||
|
||||
if not is_signed and narrow:
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4544
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4544
|
||||
# Reinstate warning check when brevitas export is fixed
|
||||
pytest.skip("Skipping checking of invalid brevitas quant setting (signed=0,narrow=1)")
|
||||
# with pytest.raises(AssertionError, match=r"Can not use narrow range.*"):
|
||||
|
||||
@@ -416,7 +416,7 @@ def check_onnx_file_dump(
|
||||
|
||||
if model_name == "KNeighborsClassifier":
|
||||
# KNN can only be compiled with small quantization bit numbers for now
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3979
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3979
|
||||
model.n_bits = 2
|
||||
|
||||
model.fit(x, y)
|
||||
@@ -446,7 +446,7 @@ def check_onnx_file_dump(
|
||||
print(f"\nExpected {model_name=}:\n{str_expected}")
|
||||
|
||||
# Test equality when it does not depend on seeds
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3266
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3266
|
||||
if not is_model_class_in_a_list(model_class, _get_sklearn_tree_models(select="RandomForest")):
|
||||
# The expected graph is usually a string and we therefore directly test if it is equal to
|
||||
# the retrieved graph's string. However, in some cases such as for TweedieRegressor models,
|
||||
|
||||
@@ -164,7 +164,7 @@ def get_n_bits_non_correctness(model_class):
|
||||
"""Get the number of bits to use for non correctness related tests."""
|
||||
|
||||
# KNN can only be compiled with small quantization bit numbers for now
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3979
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3979
|
||||
if get_model_name(model_class) == "KNeighborsClassifier":
|
||||
n_bits = 2
|
||||
else:
|
||||
@@ -206,7 +206,7 @@ def check_correctness_with_sklearn(
|
||||
|
||||
# If the model is a classifier
|
||||
# KNeighborsClassifier does not provide a predict_proba method for now
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||
if (
|
||||
is_classifier_or_partial_classifier(model)
|
||||
and get_model_name(model_class) != "KNeighborsClassifier"
|
||||
@@ -263,7 +263,7 @@ def check_double_fit(model_class, n_bits, x_1, x_2, y_1, y_2):
|
||||
"""Check double fit."""
|
||||
|
||||
if get_model_name(model_class) == "KNeighborsClassifier":
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4014
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4014
|
||||
pytest.skip(
|
||||
"Given that KNN is not accurate and the test data-set is small"
|
||||
"the y_pred1 and y_pred2 can be equal."
|
||||
@@ -420,7 +420,7 @@ def check_serialization_dump_load(model, x, use_dump_method):
|
||||
assert numpy.array_equal(y_pred_sklearn_model, y_pred_loaded_sklearn_model)
|
||||
|
||||
# Add a test to check that graphs before and after the serialization are identical
|
||||
# FIME: https://github.com/zama-ai/concrete-ml-internal/issues/4175
|
||||
# FIME: https://github.com/luxfi/concrete-ml-internal/issues/4175
|
||||
|
||||
|
||||
def check_serialization_dumps_loads(model, x, use_dump_method):
|
||||
@@ -476,7 +476,7 @@ def check_serialization_dumps_loads(model, x, use_dump_method):
|
||||
assert numpy.array_equal(y_pred_sklearn_model, y_pred_loaded_sklearn_model)
|
||||
|
||||
# Add a test to check that graphs before and after the serialization are identical
|
||||
# FIME: https://github.com/zama-ai/concrete-ml-internal/issues/4175
|
||||
# FIME: https://github.com/luxfi/concrete-ml-internal/issues/4175
|
||||
|
||||
|
||||
def check_offset(model_class, n_bits, x, y):
|
||||
@@ -502,7 +502,7 @@ def check_inference_methods(model, model_class, x, check_float_array_equal):
|
||||
# method accessible by anyone, without having any FHE implementation. As this could create some
|
||||
# confusion, a NotImplementedError is raised. This issue could be fixed by making these classes
|
||||
# not inherit from skorch.
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||
if get_model_name(model) == "NeuralNetRegressor":
|
||||
with pytest.raises(
|
||||
NotImplementedError,
|
||||
@@ -514,7 +514,7 @@ def check_inference_methods(model, model_class, x, check_float_array_equal):
|
||||
model.predict_proba(x)
|
||||
|
||||
# KNeighborsClassifier does not provide a predict_proba method for now
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||
elif get_model_name(model) == "KNeighborsClassifier":
|
||||
with pytest.raises(
|
||||
NotImplementedError,
|
||||
@@ -526,7 +526,7 @@ def check_inference_methods(model, model_class, x, check_float_array_equal):
|
||||
model.predict_proba(x)
|
||||
|
||||
# KNeighborsClassifier does not provide a kneighbors method
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4080
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4080
|
||||
with pytest.raises(
|
||||
NotImplementedError,
|
||||
match=(
|
||||
@@ -628,7 +628,7 @@ def check_separated_inference(model, fhe_circuit, x, check_float_array_equal):
|
||||
y_pred = model.post_processing(y_pred)
|
||||
|
||||
# KNeighborsClassifier does not provide a predict_proba method for now
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||
if (
|
||||
is_classifier_or_partial_classifier(model)
|
||||
and get_model_name(model) != "KNeighborsClassifier"
|
||||
@@ -705,7 +705,7 @@ def check_input_support(model_class, n_bits, default_configuration, x, y, input_
|
||||
|
||||
# Similarly, we test `predict_proba` for classifiers
|
||||
# KNeighborsClassifier does not provide a predict_proba method for now
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||
if (
|
||||
is_classifier_or_partial_classifier(model)
|
||||
and get_model_name(model_class) != "KNeighborsClassifier"
|
||||
@@ -719,7 +719,7 @@ def check_input_support(model_class, n_bits, default_configuration, x, y, input_
|
||||
|
||||
# Similarly, we test `predict_proba` for classifiers
|
||||
# KNeighborsClassifier does not provide a predict_proba method for now
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||
if (
|
||||
is_classifier_or_partial_classifier(model)
|
||||
and get_model_name(model_class) != "KNeighborsClassifier"
|
||||
@@ -797,7 +797,7 @@ def check_grid_search(model_class, x, y, scoring):
|
||||
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
|
||||
|
||||
# KNeighborsClassifier does not provide a predict_proba method for now
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||
if get_model_name(model_class) == "KNeighborsClassifier" and scoring in [
|
||||
"roc_auc",
|
||||
"average_precision",
|
||||
@@ -932,7 +932,7 @@ def check_fitted_compiled_error_raises(model_class, n_bits, x, y):
|
||||
model.predict(x)
|
||||
|
||||
# KNeighborsClassifier does not provide a predict_proba method for now
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||
if (
|
||||
is_classifier_or_partial_classifier(model_class)
|
||||
and get_model_name(model) != "KNeighborsClassifier"
|
||||
@@ -1384,7 +1384,7 @@ def check_rounding_consistency(
|
||||
|
||||
# Check that the maximum bit-width of the circuit with rounding is at most:
|
||||
# maximum bit-width (of the circuit without rounding) + 2
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4178
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4178
|
||||
|
||||
|
||||
def check_sum_for_tree_based_models(
|
||||
@@ -1706,7 +1706,7 @@ def test_inference_methods(
|
||||
|
||||
# Pipeline test sometimes fails with RandomForest models. This bug may come from Hummingbird
|
||||
# and needs further investigations
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2779
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2779
|
||||
@pytest.mark.parametrize(
|
||||
"model_class, parameters",
|
||||
get_sklearn_all_models_and_datasets(ignore="RandomForest"),
|
||||
@@ -1773,7 +1773,7 @@ def test_predict_correctness(
|
||||
"""Test prediction correctness between clear quantized and FHE simulation or execution."""
|
||||
|
||||
# KNN can only be compiled with small quantization bit numbers for now
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3979
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3979
|
||||
if n_bits > 5 and get_model_name(model_class) == "KNeighborsClassifier":
|
||||
pytest.skip("KNeighborsClassifier models can only run with 5 bits at most.")
|
||||
|
||||
@@ -1817,7 +1817,7 @@ def test_separated_inference(
|
||||
n_bits = min(N_BITS_REGULAR_BUILDS)
|
||||
|
||||
# KNN can only be compiled with small quantization bit numbers for now
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3979
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3979
|
||||
if n_bits > 5 and get_model_name(model_class) == "KNeighborsClassifier":
|
||||
pytest.skip("KNeighborsClassifier models can only run with 5 bits at most.")
|
||||
|
||||
@@ -1862,7 +1862,7 @@ def test_fitted_compiled_error_raises(
|
||||
check_fitted_compiled_error_raises(model_class, n_bits, x, y)
|
||||
|
||||
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4169
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4169
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.parametrize("model_class, parameters", MODELS_AND_DATASETS)
|
||||
@pytest.mark.parametrize(
|
||||
@@ -1898,7 +1898,7 @@ def test_p_error_simulation(
|
||||
"""Detect divergence between simulated/FHE execution and clear run."""
|
||||
|
||||
# KNeighborsClassifier does not provide a predict_proba method for now
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||
predict_function = (
|
||||
model.predict_proba
|
||||
if is_classifier_or_partial_classifier(model)
|
||||
@@ -1921,7 +1921,7 @@ def test_p_error_simulation(
|
||||
|
||||
# Skip the following if model is linear
|
||||
# Simulation and FHE differs with very high p_error on leveled circuit
|
||||
# FIXME https://github.com/zama-ai/concrete-ml-internal/issues/4343
|
||||
# FIXME https://github.com/luxfi/concrete-ml-internal/issues/4343
|
||||
if is_linear_model:
|
||||
pytest.skip("Skipping test for linear models")
|
||||
|
||||
@@ -2063,7 +2063,7 @@ def test_linear_models_have_no_tlu(
|
||||
|
||||
# This test does not check rounding at level 2
|
||||
# Additional tests for this purpose should be added in future updates
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4179
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4179
|
||||
@pytest.mark.parametrize("model_class, parameters", get_sklearn_tree_models_and_datasets())
|
||||
@pytest.mark.parametrize("n_bits", [2, 5, 8])
|
||||
def test_rounding_consistency_for_regular_models(
|
||||
@@ -2139,7 +2139,7 @@ def test_sum_for_tree_based_models(
|
||||
|
||||
|
||||
# This test should be extended to all built-in models.
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4234
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4234
|
||||
@pytest.mark.parametrize(
|
||||
"n_bits, error_message",
|
||||
[
|
||||
@@ -2168,7 +2168,7 @@ def test_invalid_n_bits_setting(model_class, n_bits, error_message):
|
||||
|
||||
|
||||
# This test should be extended to all built-in models.
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4234
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4234
|
||||
@pytest.mark.parametrize("n_bits", [5, {"op_inputs": 5}, {"op_inputs": 2, "op_leaves": 1}])
|
||||
@pytest.mark.parametrize("model_class, parameters", get_sklearn_tree_models_and_datasets())
|
||||
def test_valid_n_bits_setting(
|
||||
@@ -2230,7 +2230,7 @@ def test_error_raise_unsupported_pandas_values(model_class, bad_value, expected_
|
||||
|
||||
|
||||
# Add QNNs in this test
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4436
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4436
|
||||
@pytest.mark.parametrize(
|
||||
"model_class, parameters",
|
||||
get_sklearn_linear_models_and_datasets()
|
||||
|
||||
@@ -167,7 +167,7 @@ def train_brevitas_network_tinymnist(is_cnn, qat_bits, signed, narrow, pot_scali
|
||||
|
||||
|
||||
# This test is a known flaky
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3933
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3933
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.parametrize("qat_bits", [3])
|
||||
@pytest.mark.parametrize("signed, narrow", [(True, False), (True, True), (False, False)])
|
||||
@@ -234,7 +234,7 @@ def test_brevitas_tinymnist_cnn(
|
||||
# Accept, at most, 1% examples that are classified differently (currently 5)
|
||||
# For now, the correctness test has been disabled as it was too flaky, it should however be put
|
||||
# back at one point
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2550
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2550
|
||||
# assert abs(fhe_simulation_correct - torch_correct) <= numpy.ceil(0.01 * len(y_test))
|
||||
|
||||
assert fhe_s_correct >= 0
|
||||
@@ -245,7 +245,7 @@ def test_brevitas_tinymnist_cnn(
|
||||
|
||||
# Note that this test is currently disabled until the pytorch dtype issue is found
|
||||
# and all mismatches between Concrete ML and Brevitas are fixed
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2373
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2373
|
||||
@pytest.mark.parametrize(
|
||||
"n_layers",
|
||||
[3],
|
||||
@@ -496,7 +496,7 @@ def test_brevitas_constant_folding(default_configuration):
|
||||
|
||||
|
||||
# This test is a known flaky
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4356
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4356
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.parametrize("manual_rounding", [None, 3])
|
||||
@pytest.mark.parametrize("power_of_two", [True, False])
|
||||
|
||||
@@ -52,7 +52,7 @@ def compile_and_test_keras(
|
||||
|
||||
|
||||
# We should also have some correctness tests
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2749
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2749
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -398,13 +398,13 @@ def accuracy_test_rounding(
|
||||
for key, module in compiled_modules.items():
|
||||
|
||||
# low bit-width rounding is not behaving as expected with new simulation
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4331
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4331
|
||||
if "low" not in key:
|
||||
check_is_good_execution_for_cml_vs_circuit(x_test, module, simulate=simulate)
|
||||
|
||||
# FIXME: The following MSE comparison is commented out due to instability issues.
|
||||
# We will investigate a better way to assess the rounding feature's performance.
|
||||
# https://github.com/zama-ai/concrete-ml-internal/issues/3662
|
||||
# https://github.com/luxfi/concrete-ml-internal/issues/3662
|
||||
# mse_results = {
|
||||
# key: numpy.mean(numpy.square(numpy.subtract(results['original'], result_list)))
|
||||
# for key, result_list in results.items()
|
||||
@@ -416,7 +416,7 @@ def accuracy_test_rounding(
|
||||
|
||||
|
||||
# This test is a known flaky
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3429
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3429
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.parametrize(
|
||||
"activation_function",
|
||||
@@ -481,7 +481,7 @@ def test_compile_torch_or_onnx_networks(
|
||||
|
||||
|
||||
# This test is a known flaky
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3660
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3660
|
||||
@pytest.mark.flaky
|
||||
@pytest.mark.parametrize(
|
||||
"activation_function",
|
||||
@@ -564,7 +564,7 @@ def test_compile_torch_or_onnx_conv_networks( # pylint: disable=unused-argument
|
||||
pytest.param(nn.GELU, id="GELU"),
|
||||
pytest.param(nn.LogSigmoid, id="LogSigmoid"),
|
||||
# Some issues are still encountered with some activations
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/335
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/335
|
||||
#
|
||||
# Other problems, certainly related to tests:
|
||||
# Required positional arguments: 'embed_dim' and 'num_heads' and fails with a partial
|
||||
@@ -699,7 +699,7 @@ def test_compile_brevitas_qat(
|
||||
|
||||
|
||||
# Update this test to align with Concrete's simulation fix.
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4578
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4578
|
||||
@pytest.mark.xfail
|
||||
@pytest.mark.parametrize(
|
||||
"model_class, expected_onnx_str",
|
||||
@@ -1031,7 +1031,7 @@ def test_shape_operations_net(
|
||||
model = model_class(is_qat)
|
||||
|
||||
# Shape transformation do not support >1 example in the inputset
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3871
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3871
|
||||
inputset = numpy.random.uniform(size=(1, n_channels, 2, 2))
|
||||
|
||||
if is_qat:
|
||||
|
||||
@@ -62,7 +62,7 @@ from concrete.ml.torch import NumpyModule
|
||||
pytest.param(nn.LogSigmoid, id="LogSigmoid"),
|
||||
pytest.param(nn.GELU, id="GELU"),
|
||||
# Some issues are still encountered with some activations
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/335
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/335
|
||||
#
|
||||
# Other problems, certainly related to tests:
|
||||
# Required positional arguments: 'embed_dim' and 'num_heads' and fails with a partial
|
||||
|
||||
@@ -110,7 +110,7 @@ def test_sgd_training_manual(
|
||||
): # pylint: disable=unused-argument
|
||||
"""Trains a logistic regression with SGD in torch and quantized."""
|
||||
# Train on the bias when multi output is available in concrete
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4131
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4131
|
||||
|
||||
print("test_sgd_training_manual", get_device)
|
||||
|
||||
|
||||
@@ -67,7 +67,7 @@ DATASETS_ARGS = {
|
||||
),
|
||||
},
|
||||
# Separate FMNIST from CIFAR directory
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3552
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3552
|
||||
"FashionMNIST": {
|
||||
"dataset": datasets.FashionMNIST,
|
||||
"mean": (0.2859),
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ KEYGEN_CACHE_DIR = CURRENT_DIR.joinpath(".keycache")
|
||||
|
||||
# Add MPS (for macOS with Apple Silicon or AMD GPUs) support when error is fixed. For now, we
|
||||
# observe a decrease in torch's top1 accuracy when using MPS devices
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3953
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3953
|
||||
|
||||
# For PyTorch operations, we use CPU (simpler and avoids device mismatch issues)
|
||||
DEVICE = "cpu"
|
||||
|
||||
+1
-1
@@ -73,7 +73,7 @@ def main(args):
|
||||
|
||||
# Add MPS (for macOS with Apple Silicon or AMD GPUs) support when error is fixed. For now, we
|
||||
# observe a decrease in torch's top1 accuracy when using MPS devices
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3953
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3953
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
compilation_device = "cuda" if concrete.compiler.check_gpu_available() else "cpu"
|
||||
|
||||
|
||||
@@ -201,7 +201,7 @@ class Trainer(object):
|
||||
else:
|
||||
# Add MPS (for macOS with Apple Silicon or AMD GPUs) support when error is fixed. For
|
||||
# now, we observe a decrease in torch's top1 accuracy when using MPS devices
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3953
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3953
|
||||
self.device = "cpu"
|
||||
|
||||
self.device = torch.device(self.device)
|
||||
|
||||
@@ -54,7 +54,7 @@ def train(dev_folder="./dev"):
|
||||
|
||||
# Add MPS (for macOS with Apple Silicon or AMD GPUs) support when error is fixed. For now, we
|
||||
# observe a decrease in torch's top1 accuracy when using MPS devices
|
||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3953
|
||||
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3953
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
|
||||
# # Load the tokenizer (converts text to tokens)
|
||||
@@ -167,7 +167,7 @@ def train(dev_folder="./dev"):
|
||||
print(f"Compilation time: {end - start:.4f} seconds")
|
||||
|
||||
# Write a custom example and predict in FHE
|
||||
tested_tweet = ["AirFrance is awesome, almost as much as Zama!"]
|
||||
tested_tweet = ["AirFrance is awesome, almost as much as Lux!"]
|
||||
X_tested_tweet = text_to_tensor(tested_tweet, transformer_model, tokenizer, device)
|
||||
clear_proba = best_model.predict_proba(X_tested_tweet)
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ It takes as input a folder with:
|
||||
It builds a Docker image and spawns a Docker container that runs the server.
|
||||
|
||||
This module is untested as it would require to first build the release Docker image.
|
||||
FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3347
|
||||
FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3347
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
||||
@@ -425,7 +425,7 @@ class DualArray:
|
||||
|
||||
# Concrete-Python does not support numpy.array_split and numpy.take so we need to build a custom
|
||||
# split method instead
|
||||
# FIXME: https://github.com/zama-ai/concrete-internal/issues/329
|
||||
# FIXME: https://github.com/luxfi/concrete-internal/issues/329
|
||||
def enc_split(self, n: int, axis: int, key: str) -> Tuple[DualArray]:
|
||||
"""Split the arrays in n parts along a given axis."""
|
||||
self_int_array = self._ensure_quantized(key=f"{key}_self")
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
__name__ = "concrete-ml-extensions"
|
||||
__author__ = "Zama"
|
||||
__author__ = "Lux Network"
|
||||
__all__ = ["concrete-ml-extensions"]
|
||||
__version__ = "0.2.0"
|
||||
|
||||
|
||||
Executable
+272
@@ -0,0 +1,272 @@
|
||||
#!/bin/bash
|
||||
# Migrate Zama FHE repos to Lux FHE stack
|
||||
# Forks to github.com/luxfi, strips Zama references, updates imports
|
||||
|
||||
set -e
|
||||
|
||||
ZAMA_DIR="$HOME/work/zama"
|
||||
LUX_FHE_DIR="$HOME/work/lux/fhe"
|
||||
LUXFI_ORG="luxfi"
|
||||
|
||||
# Color output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
log() { echo -e "${GREEN}[+]${NC} $1"; }
|
||||
warn() { echo -e "${YELLOW}[!]${NC} $1"; }
|
||||
error() { echo -e "${RED}[-]${NC} $1"; }
|
||||
|
||||
# Repos to migrate with their new names
|
||||
declare -A MIGRATE_REPOS=(
|
||||
# Core libraries
|
||||
["concrete-compiler-internal-llvm-project"]="fhe-compiler"
|
||||
["concrete-ntt"]="fhe-ntt"
|
||||
["concrete-fft"]="fhe-fft"
|
||||
|
||||
# ML
|
||||
["concrete-ml"]="fhe-ml"
|
||||
["concrete-ml-extensions"]="fhe-ml-extensions"
|
||||
["concrete-ml-processing-rs"]="fhe-ml-rs"
|
||||
|
||||
# Templates & dApps
|
||||
["dapps"]="fhe-dapps"
|
||||
["fhevm-hardhat-template"]="fhe-hardhat-template"
|
||||
["fhevm-next-template"]="fhe-next-template"
|
||||
["fhevm-react-template"]="fhe-react-template"
|
||||
["fhevm-vue-template"]="fhe-vue-template"
|
||||
["fhevm-remix-plugin"]="fhe-remix-plugin"
|
||||
["fhevm-workshop"]="fhe-workshop"
|
||||
["fhevm-test-suite"]="fhe-test-suite"
|
||||
["fhevm-mocks"]="fhe-mocks"
|
||||
|
||||
# Infrastructure
|
||||
["fhevm-contracts"]="fhe-contracts"
|
||||
["fhevm-go"]="fhe-go"
|
||||
["fhevm-solidity"]="fhe-solidity"
|
||||
["fhevm-decryptions-db"]="fhe-decryptions-db"
|
||||
["fhevm-tfhe-cli"]="fhe-cli"
|
||||
["go-ethereum-coprocessor"]="fhe-geth"
|
||||
["relayer-sdk"]="fhe-relayer"
|
||||
|
||||
# Research & Docs
|
||||
["tfhe-rs-handbook"]="fhe-handbook"
|
||||
["verifiable-fhe-paper"]="fhe-papers"
|
||||
["threshold-fhe"]="fhe-threshold"
|
||||
|
||||
# Demos
|
||||
["fhe_ios_demo"]="fhe-ios"
|
||||
["fhe-biometrics"]="fhe-biometrics"
|
||||
)
|
||||
|
||||
# Skip these (Zama-specific, already have, or third-party)
|
||||
SKIP_REPOS=(
|
||||
"tfhe-rs" # Use luxfi/fhe instead
|
||||
"concrete" # Zama main product
|
||||
"awesome-zama" # Promotional
|
||||
"bounty-ecdsa-signature"
|
||||
"bounty-matrix-inversion"
|
||||
"bounty-program"
|
||||
"ci-templates"
|
||||
"fhenix" # Third party
|
||||
"HElib" # IBM
|
||||
"SEAL" # Microsoft
|
||||
"openfhe-development"
|
||||
"slab-github-runner"
|
||||
"terraform-mpc-modules"
|
||||
"tfhe-backward-compat-data"
|
||||
"zbc-go-ethereum"
|
||||
"hpu_fpga"
|
||||
"hw_regmap"
|
||||
"ocp-fhe"
|
||||
"poc-prime-match"
|
||||
"progress-tracker-python"
|
||||
"copz25-code"
|
||||
"blockchain-wallet-exporter"
|
||||
)
|
||||
|
||||
# Files/patterns to strip Zama references from
|
||||
ZAMA_PATTERNS=(
|
||||
"zama.ai"
|
||||
"zama-ai"
|
||||
"Zama"
|
||||
"ZAMA"
|
||||
"tfhe-rs"
|
||||
"github.com/zama-ai"
|
||||
"@zama-ai"
|
||||
)
|
||||
|
||||
# Replacement mappings
|
||||
declare -A REPLACEMENTS=(
|
||||
["zama.ai"]="lux.network"
|
||||
["zama-ai"]="luxfi"
|
||||
["Zama"]="Lux"
|
||||
["ZAMA"]="LUX"
|
||||
["tfhe-rs"]="luxfi/fhe"
|
||||
["github.com/zama-ai"]="github.com/luxfi"
|
||||
["@zama-ai"]="@luxfi"
|
||||
["concrete-ml"]="fhe-ml"
|
||||
["concrete-ntt"]="fhe-ntt"
|
||||
["concrete-fft"]="fhe-fft"
|
||||
["fhevm-"]="fhe-"
|
||||
)
|
||||
|
||||
fork_repo() {
|
||||
local src_name="$1"
|
||||
local dst_name="$2"
|
||||
|
||||
log "Forking $src_name -> $dst_name"
|
||||
|
||||
# Check if already exists
|
||||
if gh repo view "$LUXFI_ORG/$dst_name" &>/dev/null; then
|
||||
warn "Repo $LUXFI_ORG/$dst_name already exists, skipping fork"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Check if source exists locally
|
||||
if [[ ! -d "$ZAMA_DIR/$src_name" ]]; then
|
||||
error "Source $ZAMA_DIR/$src_name not found"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Create new repo
|
||||
gh repo create "$LUXFI_ORG/$dst_name" --public --description "Lux FHE: $(echo $dst_name | sed 's/-/ /g')" || {
|
||||
warn "Could not create repo, may already exist"
|
||||
}
|
||||
|
||||
# Clone, strip refs, push
|
||||
local tmp_dir="/tmp/migrate-$$-$dst_name"
|
||||
cp -r "$ZAMA_DIR/$src_name" "$tmp_dir"
|
||||
cd "$tmp_dir"
|
||||
|
||||
# Remove old git history
|
||||
rm -rf .git
|
||||
git init
|
||||
|
||||
# Strip Zama references
|
||||
strip_zama_refs "$tmp_dir"
|
||||
|
||||
# Commit and push
|
||||
git add -A
|
||||
git commit -m "Initial import from Zama (Apache 2.0), migrated to Lux FHE stack"
|
||||
git branch -M main
|
||||
git remote add origin "git@github.com:$LUXFI_ORG/$dst_name.git"
|
||||
git push -u origin main --force || warn "Push failed for $dst_name"
|
||||
|
||||
cd -
|
||||
rm -rf "$tmp_dir"
|
||||
|
||||
log "Successfully migrated $src_name -> $dst_name"
|
||||
}
|
||||
|
||||
strip_zama_refs() {
|
||||
local dir="$1"
|
||||
|
||||
log "Stripping Zama references from $dir"
|
||||
|
||||
# Find all text files
|
||||
find "$dir" -type f \( -name "*.py" -o -name "*.rs" -o -name "*.go" -o -name "*.ts" -o -name "*.js" \
|
||||
-o -name "*.sol" -o -name "*.md" -o -name "*.json" -o -name "*.yaml" -o -name "*.yml" \
|
||||
-o -name "*.toml" -o -name "Makefile" -o -name "*.sh" -o -name "*.txt" \
|
||||
-o -name "Cargo.toml" -o -name "package.json" -o -name "pyproject.toml" \) 2>/dev/null | while read -r file; do
|
||||
|
||||
# Apply replacements
|
||||
for pattern in "${!REPLACEMENTS[@]}"; do
|
||||
replacement="${REPLACEMENTS[$pattern]}"
|
||||
if grep -q "$pattern" "$file" 2>/dev/null; then
|
||||
sed -i '' "s|$pattern|$replacement|g" "$file" 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
# Update README files
|
||||
find "$dir" -name "README.md" -type f 2>/dev/null | while read -r readme; do
|
||||
# Add Lux header
|
||||
if ! grep -q "Lux FHE" "$readme" 2>/dev/null; then
|
||||
sed -i '' '1s/^/# Lux FHE\n\nPart of the Lux FHE stack. See https:\/\/github.com\/luxfi\/fhe\n\n/' "$readme" 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
update_imports() {
|
||||
local dir="$1"
|
||||
|
||||
log "Updating imports in $dir"
|
||||
|
||||
# Python imports
|
||||
find "$dir" -name "*.py" -type f 2>/dev/null | xargs -I {} sed -i '' \
|
||||
-e 's/from concrete\./from luxfhe./g' \
|
||||
-e 's/import concrete/import luxfhe/g' \
|
||||
-e 's/concrete-ml/fhe-ml/g' \
|
||||
{} 2>/dev/null || true
|
||||
|
||||
# Rust imports
|
||||
find "$dir" -name "*.rs" -o -name "Cargo.toml" 2>/dev/null | xargs -I {} sed -i '' \
|
||||
-e 's/tfhe = /fhe = { git = "https:\/\/github.com\/luxfi\/fhe" } # /g' \
|
||||
-e 's/concrete_/luxfhe_/g' \
|
||||
{} 2>/dev/null || true
|
||||
|
||||
# TypeScript/JavaScript
|
||||
find "$dir" -name "*.ts" -o -name "*.js" -o -name "package.json" 2>/dev/null | xargs -I {} sed -i '' \
|
||||
-e 's/@zama-ai\/fhevm/@luxfi\/fhe/g' \
|
||||
-e 's/fhevm/luxfhe/g' \
|
||||
{} 2>/dev/null || true
|
||||
|
||||
# Go imports
|
||||
find "$dir" -name "*.go" -o -name "go.mod" 2>/dev/null | xargs -I {} sed -i '' \
|
||||
-e 's|github.com/zama-ai/|github.com/luxfi/|g' \
|
||||
{} 2>/dev/null || true
|
||||
}
|
||||
|
||||
# Main migration
|
||||
main() {
|
||||
log "Starting Zama -> Lux FHE migration"
|
||||
log "Source: $ZAMA_DIR"
|
||||
log "Org: $LUXFI_ORG"
|
||||
echo
|
||||
|
||||
# Check prerequisites
|
||||
if ! command -v gh &>/dev/null; then
|
||||
error "GitHub CLI (gh) not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! gh auth status &>/dev/null; then
|
||||
error "Not logged into GitHub CLI"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Migrate each repo
|
||||
for src_name in "${!MIGRATE_REPOS[@]}"; do
|
||||
dst_name="${MIGRATE_REPOS[$src_name]}"
|
||||
|
||||
echo "---"
|
||||
fork_repo "$src_name" "$dst_name" || warn "Failed to migrate $src_name"
|
||||
done
|
||||
|
||||
log "Migration complete!"
|
||||
echo
|
||||
log "Next steps:"
|
||||
echo " 1. Review each repo and fix any remaining Zama references"
|
||||
echo " 2. Update CI/CD pipelines"
|
||||
echo " 3. Update documentation"
|
||||
echo " 4. Test integrations with luxfi/fhe"
|
||||
}
|
||||
|
||||
# Run with specific repo or all
|
||||
if [[ $# -eq 1 ]]; then
|
||||
src="$1"
|
||||
if [[ -v "MIGRATE_REPOS[$src]" ]]; then
|
||||
fork_repo "$src" "${MIGRATE_REPOS[$src]}"
|
||||
else
|
||||
error "Unknown repo: $src"
|
||||
echo "Available repos:"
|
||||
for r in "${!MIGRATE_REPOS[@]}"; do
|
||||
echo " $r -> ${MIGRATE_REPOS[$r]}"
|
||||
done
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
main
|
||||
fi
|
||||
@@ -4,10 +4,10 @@ The `fhevm` Command-Line Interface (CLI) tool provides a simple and efficient wa
|
||||
|
||||
## Installation
|
||||
|
||||
Ensure you have [Node.js](https://nodejs.org/) installed on your system before proceeding. Then, globally install the `@zama-fhe/relayer-sdk` package to enable the CLI tool:
|
||||
Ensure you have [Node.js](https://nodejs.org/) installed on your system before proceeding. Then, globally install the `@luxfhe/relayer-sdk` package to enable the CLI tool:
|
||||
|
||||
```bash
|
||||
npm install -g @zama-fhe/relayer-sdk
|
||||
npm install -g @luxfhe/relayer-sdk
|
||||
```
|
||||
|
||||
Once installed, you can access the CLI using the `relayer` command. Verify the installation and explore available commands using:
|
||||
|
||||
@@ -9,7 +9,7 @@ Public decryption can be done using the Relayer HTTP endpoint.
|
||||
Calling the public decryption endpoint of the Relayer can be done easily using the following code snippet.
|
||||
|
||||
{% hint style="info" %}
|
||||
The total bit length of all ciphertexts being decrypted in a single request must not exceed 2048 bits. Each encrypted type has a specific bit length, for instance `euint8` uses 8 bits and `euint16` uses 16 bits. For the full list of encrypted types and their corresponding bit lengths, refer to the [encrypted types documentation](https://docs.zama.org/protocol/solidity-guides/smart-contract/types#list-of-encrypted-types).
|
||||
The total bit length of all ciphertexts being decrypted in a single request must not exceed 2048 bits. Each encrypted type has a specific bit length, for instance `euint8` uses 8 bits and `euint16` uses 16 bits. For the full list of encrypted types and their corresponding bit lengths, refer to the [encrypted types documentation](https://docs.lux.network/protocol/solidity-guides/smart-contract/types#list-of-encrypted-types).
|
||||
{% endhint %}
|
||||
|
||||
```ts
|
||||
|
||||
@@ -48,16 +48,16 @@ For more details on the topic please refer to [the ACL documentation](https://do
|
||||
|
||||
## Step 2: decrypt the ciphertext
|
||||
|
||||
Using those ciphertext handles, user decryption is performed client-side using the `@zama-fhe/relayer-sdk` library.
|
||||
Using those ciphertext handles, user decryption is performed client-side using the `@luxfhe/relayer-sdk` library.
|
||||
The `userDecrypt` function takes a **list of handles**, allowing you to decrypt multiple ciphertexts in a single request. In this example, provide just one handle.
|
||||
The user needs to have created an instance object prior to that (for more context see [the relayer-sdk setup page](./initialization.md)).
|
||||
|
||||
{% hint style="info" %}
|
||||
The total bit length of all ciphertexts being decrypted in a single request must not exceed 2048 bits. Each encrypted type has a specific bit length, for instance `euint8` uses 8 bits and `euint16` uses 16 bits. For the full list of encrypted types and their corresponding bit lengths, refer to the [encrypted types documentation](https://docs.zama.org/protocol/solidity-guides/smart-contract/types#list-of-encrypted-types).
|
||||
The total bit length of all ciphertexts being decrypted in a single request must not exceed 2048 bits. Each encrypted type has a specific bit length, for instance `euint8` uses 8 bits and `euint16` uses 16 bits. For the full list of encrypted types and their corresponding bit lengths, refer to the [encrypted types documentation](https://docs.lux.network/protocol/solidity-guides/smart-contract/types#list-of-encrypted-types).
|
||||
{% endhint %}
|
||||
|
||||
```ts
|
||||
// instance: [`FhevmInstance`] from `zama-fhe/relayer-sdk`
|
||||
// instance: [`FhevmInstance`] from `luxfhe/relayer-sdk`
|
||||
// signer: [`Signer`] from ethers (could a [`Wallet`])
|
||||
// ciphertextHandle: [`string`]
|
||||
// contractAddress: [`string`]
|
||||
|
||||
@@ -54,7 +54,7 @@ resolve: {
|
||||
|
||||
**Cause:** The library may not bundle correctly with certain frameworks, leading to errors during the build or runtime process.
|
||||
|
||||
**Possible solutions:** Use the [prebundled version available](./webapp.md) with `@zama-fhe/relayer-sdk/bundle`. Embed the library with a `<script>` tag and initialize it as shown below:
|
||||
**Possible solutions:** Use the [prebundled version available](./webapp.md) with `@luxfhe/relayer-sdk/bundle`. Embed the library with a `<script>` tag and initialize it as shown below:
|
||||
|
||||
```javascript
|
||||
const start = async () => {
|
||||
|
||||
+13
@@ -481,6 +481,19 @@ func (eval *ShortIntEvaluator) addCiphertexts(ct1, ct2 *rlwe.Ciphertext) *rlwe.C
|
||||
return result
|
||||
}
|
||||
|
||||
// scaleCiphertext scales a ciphertext by a scalar value
|
||||
func (eval *ShortIntEvaluator) scaleCiphertext(ct *rlwe.Ciphertext, scalar float64) *rlwe.Ciphertext {
|
||||
result := ct.CopyNew()
|
||||
// Scale by multiplying each coefficient
|
||||
for i := range result.Value {
|
||||
coeffs := result.Value[i].Coeffs
|
||||
for j := range coeffs[0] {
|
||||
coeffs[0][j] = uint64(float64(coeffs[0][j]) * scalar)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// Add adds two shortints (with modular wrap)
|
||||
func (eval *ShortIntEvaluator) Add(a, b *ShortInt) (*ShortInt, error) {
|
||||
if a.msgSpace != b.msgSpace {
|
||||
|
||||
Reference in New Issue
Block a user