mirror of
https://github.com/luxfi/fhe.git
synced 2026-07-26 23:16:08 +00:00
Complete Lux FHE rebrand
This commit is contained in:
@@ -431,7 +431,7 @@ function reveal(bytes32 requestId) external view returns (bytes memory result, b
|
|||||||
- rng-game, rps-game, secret-santa, voting
|
- rng-game, rps-game, secret-santa, voting
|
||||||
|
|
||||||
**⚠️ Not using @luxfi/contracts:**
|
**⚠️ Not using @luxfi/contracts:**
|
||||||
- `dapps/` - Uses `@fhevm/solidity` (Zama's library)
|
- `dapps/` - Uses `@fhevm/solidity` (lux's library)
|
||||||
- `poker/`, `kuhn-poker/` - Custom implementation
|
- `poker/`, `kuhn-poker/` - Custom implementation
|
||||||
- `ticketing/`, `tickets/` - Package manager issues
|
- `ticketing/`, `tickets/` - Package manager issues
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,194 @@
|
|||||||
|
# Lux FHE
|
||||||
|
|
||||||
|
[](https://pkg.go.dev/github.com/luxfi/fhe)
|
||||||
|
[](https://github.com/luxfi/fhe/actions/workflows/ci.yml)
|
||||||
|
|
||||||
|
Fully Homomorphic Encryption (FHE) library for the Lux Network.
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
- **TFHE** - Fast Boolean operations on encrypted data
|
||||||
|
- **CKKS** - Approximate arithmetic on encrypted vectors
|
||||||
|
- **GPU Acceleration** - Metal and CUDA support via luxfi/gpu
|
||||||
|
- **Lazy Carry Propagation** - Efficient integer arithmetic
|
||||||
|
- **Multi-Party Computation** - Threshold decryption support
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go get github.com/luxfi/fhe
|
||||||
|
```
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
### TFHE Boolean Gates
|
||||||
|
|
||||||
|
```go
|
||||||
|
import "github.com/luxfi/fhe"
|
||||||
|
|
||||||
|
// Generate keys
|
||||||
|
params := fhe.DefaultTFHEParams()
|
||||||
|
keys, _ := fhe.GenerateTFHEKeys(params)
|
||||||
|
|
||||||
|
// Encrypt bits
|
||||||
|
ct1, _ := fhe.TFHEEncrypt(keys, true)
|
||||||
|
ct2, _ := fhe.TFHEEncrypt(keys, false)
|
||||||
|
|
||||||
|
// Homomorphic operations
|
||||||
|
ctAnd, _ := fhe.TFHEAnd(keys, ct1, ct2)
|
||||||
|
ctOr, _ := fhe.TFHEOr(keys, ct1, ct2)
|
||||||
|
ctNot, _ := fhe.TFHENot(keys, ct1)
|
||||||
|
|
||||||
|
// Decrypt
|
||||||
|
result, _ := fhe.TFHEDecrypt(keys, ctAnd) // false
|
||||||
|
```
|
||||||
|
|
||||||
|
### Integer Operations
|
||||||
|
|
||||||
|
```go
|
||||||
|
import "github.com/luxfi/fhe"
|
||||||
|
|
||||||
|
// Encrypt integers
|
||||||
|
a, _ := fhe.EncryptInt(keys, 42, 8) // 8-bit integer
|
||||||
|
b, _ := fhe.EncryptInt(keys, 10, 8)
|
||||||
|
|
||||||
|
// Arithmetic
|
||||||
|
sum, _ := fhe.IntAdd(keys, a, b)
|
||||||
|
diff, _ := fhe.IntSub(keys, a, b)
|
||||||
|
prod, _ := fhe.IntMul(keys, a, b)
|
||||||
|
|
||||||
|
// Comparison
|
||||||
|
isGreater, _ := fhe.IntGreaterThan(keys, a, b)
|
||||||
|
```
|
||||||
|
|
||||||
|
### CKKS Approximate Arithmetic
|
||||||
|
|
||||||
|
```go
|
||||||
|
import "github.com/luxfi/fhe"
|
||||||
|
|
||||||
|
// Setup CKKS
|
||||||
|
params := fhe.DefaultCKKSParams()
|
||||||
|
keys, _ := fhe.GenerateCKKSKeys(params)
|
||||||
|
|
||||||
|
// Encrypt vectors
|
||||||
|
values := []float64{1.0, 2.0, 3.0, 4.0}
|
||||||
|
ct, _ := fhe.CKKSEncrypt(keys, values)
|
||||||
|
|
||||||
|
// Operations
|
||||||
|
scaled, _ := fhe.CKKSMulScalar(ct, 2.5)
|
||||||
|
rotated, _ := fhe.CKKSRotate(keys, ct, 1)
|
||||||
|
|
||||||
|
// Decrypt
|
||||||
|
result, _ := fhe.CKKSDecrypt(keys, scaled)
|
||||||
|
```
|
||||||
|
|
||||||
|
## API Reference
|
||||||
|
|
||||||
|
### Key Generation
|
||||||
|
|
||||||
|
| Function | Description |
|
||||||
|
|----------|-------------|
|
||||||
|
| `GenerateTFHEKeys(params)` | Generate TFHE key set |
|
||||||
|
| `GenerateCKKSKeys(params)` | Generate CKKS key set |
|
||||||
|
|
||||||
|
### TFHE Operations
|
||||||
|
|
||||||
|
| Function | Description |
|
||||||
|
|----------|-------------|
|
||||||
|
| `TFHEEncrypt(keys, bit)` | Encrypt a bit |
|
||||||
|
| `TFHEDecrypt(keys, ct)` | Decrypt a ciphertext |
|
||||||
|
| `TFHEAnd(keys, a, b)` | Homomorphic AND |
|
||||||
|
| `TFHEOr(keys, a, b)` | Homomorphic OR |
|
||||||
|
| `TFHENot(keys, a)` | Homomorphic NOT |
|
||||||
|
| `TFHENand(keys, a, b)` | Homomorphic NAND |
|
||||||
|
| `TFHEXor(keys, a, b)` | Homomorphic XOR |
|
||||||
|
| `TFHEBootstrap(keys, ct)` | Refresh ciphertext noise |
|
||||||
|
|
||||||
|
### Integer Operations
|
||||||
|
|
||||||
|
| Function | Description |
|
||||||
|
|----------|-------------|
|
||||||
|
| `EncryptInt(keys, value, bits)` | Encrypt integer |
|
||||||
|
| `DecryptInt(keys, ct)` | Decrypt integer |
|
||||||
|
| `IntAdd(keys, a, b)` | Addition |
|
||||||
|
| `IntSub(keys, a, b)` | Subtraction |
|
||||||
|
| `IntMul(keys, a, b)` | Multiplication |
|
||||||
|
| `IntGreaterThan(keys, a, b)` | Comparison |
|
||||||
|
|
||||||
|
### CKKS Operations
|
||||||
|
|
||||||
|
| Function | Description |
|
||||||
|
|----------|-------------|
|
||||||
|
| `CKKSEncrypt(keys, values)` | Encrypt vector |
|
||||||
|
| `CKKSDecrypt(keys, ct)` | Decrypt vector |
|
||||||
|
| `CKKSAdd(a, b)` | Vector addition |
|
||||||
|
| `CKKSMul(keys, a, b)` | Vector multiplication |
|
||||||
|
| `CKKSRotate(keys, ct, steps)` | Rotate slots |
|
||||||
|
|
||||||
|
## GPU Acceleration
|
||||||
|
|
||||||
|
Enable GPU acceleration via the luxfi/gpu package:
|
||||||
|
|
||||||
|
```go
|
||||||
|
import (
|
||||||
|
"github.com/luxfi/fhe"
|
||||||
|
"github.com/luxfi/gpu"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Initialize GPU session
|
||||||
|
sess, _ := gpu.DefaultSession()
|
||||||
|
|
||||||
|
// Use GPU-accelerated FHE
|
||||||
|
fheOps := sess.FHE()
|
||||||
|
result, _ := fheOps.NTTForward(poly, modulus)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Examples
|
||||||
|
|
||||||
|
See the `examples/` directory for complete examples:
|
||||||
|
|
||||||
|
- `examples/tfhe_basic/` - Basic TFHE operations
|
||||||
|
- `examples/integer_ops/` - Integer arithmetic
|
||||||
|
- `examples/ckks_ml/` - Machine learning on encrypted data
|
||||||
|
- `examples/multiparty/` - Threshold decryption
|
||||||
|
|
||||||
|
## Documentation
|
||||||
|
|
||||||
|
- [API Reference](https://pkg.go.dev/github.com/luxfi/fhe)
|
||||||
|
- [Workshop Materials](docs/workshop/)
|
||||||
|
- [Resources](docs/resources/)
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Run all tests
|
||||||
|
go test ./...
|
||||||
|
|
||||||
|
# With GPU
|
||||||
|
CGO_ENABLED=1 go test ./...
|
||||||
|
|
||||||
|
# Benchmarks
|
||||||
|
go test -bench=. ./...
|
||||||
|
```
|
||||||
|
|
||||||
|
## Performance
|
||||||
|
|
||||||
|
Performance varies by operation and key parameters:
|
||||||
|
|
||||||
|
| Operation | Time (128-bit security) |
|
||||||
|
|-----------|-------------------------|
|
||||||
|
| TFHE Gate | ~10ms |
|
||||||
|
| Bootstrap | ~50ms |
|
||||||
|
| CKKS Add | <1ms |
|
||||||
|
| CKKS Mul | ~5ms |
|
||||||
|
| NTT (N=2^16) | ~1ms (GPU) |
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
BSD 3-Clause. See [LICENSE](LICENSE).
|
||||||
|
|
||||||
|
## Links
|
||||||
|
|
||||||
|
- [Lux FHE Documentation](https://docs.luxfhe.ai)
|
||||||
|
- [Research Papers](docs/resources/README.md)
|
||||||
|
- [GitHub Issues](https://github.com/luxfi/fhe/issues)
|
||||||
+6
-9
@@ -473,6 +473,7 @@ func (eval *BitwiseEvaluator) Eq(a, b *BitCiphertext) (*Ciphertext, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Lt returns encrypted 1 if a < b, 0 otherwise (unsigned)
|
// Lt returns encrypted 1 if a < b, 0 otherwise (unsigned)
|
||||||
|
// Optimized using CMPCOMBINE gate to reduce bootstraps from ~37 to ~23 for 8-bit
|
||||||
func (eval *BitwiseEvaluator) Lt(a, b *BitCiphertext) (*Ciphertext, error) {
|
func (eval *BitwiseEvaluator) Lt(a, b *BitCiphertext) (*Ciphertext, error) {
|
||||||
if a.numBits != b.numBits {
|
if a.numBits != b.numBits {
|
||||||
return nil, fmt.Errorf("bit count mismatch")
|
return nil, fmt.Errorf("bit count mismatch")
|
||||||
@@ -487,9 +488,8 @@ func (eval *BitwiseEvaluator) Lt(a, b *BitCiphertext) (*Ciphertext, error) {
|
|||||||
|
|
||||||
// Start from MSB
|
// Start from MSB
|
||||||
for i := numBits - 1; i >= 0; i-- {
|
for i := numBits - 1; i >= 0; i-- {
|
||||||
// a[i] < b[i]: NOT(a[i]) AND b[i]
|
// a[i] < b[i]: NOT(a[i]) AND b[i] = ANDNY(a[i], b[i])
|
||||||
notA := eval.eval.NOT(a.bits[i])
|
bitLt, err := eval.eval.ANDNY(a.bits[i], b.bits[i])
|
||||||
bitLt, err := eval.eval.AND(notA, b.bits[i])
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -504,12 +504,9 @@ func (eval *BitwiseEvaluator) Lt(a, b *BitCiphertext) (*Ciphertext, error) {
|
|||||||
isLess = bitLt
|
isLess = bitLt
|
||||||
isEqual = bitEq
|
isEqual = bitEq
|
||||||
} else {
|
} else {
|
||||||
// isLess = isLess OR (isEqual AND bitLt)
|
// Use CMPCOMBINE to compute: isLess OR (isEqual AND bitLt)
|
||||||
eqAndLt, err := eval.eval.AND(isEqual, bitLt)
|
// This replaces 2 bootstraps (AND + OR) with 1 bootstrap
|
||||||
if err != nil {
|
isLess, err = eval.eval.CMPCOMBINE(isLess, isEqual, bitLt)
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
isLess, err = eval.eval.OR(isLess, eqAndLt)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ Install LuxFHE Devnet in MetaMask:
|
|||||||
- Network name: LuxFHE Devnet
|
- Network name: LuxFHE Devnet
|
||||||
- RPC URL: https://devnet.luxfhe.ai
|
- RPC URL: https://devnet.luxfhe.ai
|
||||||
- Chain ID: 8009
|
- Chain ID: 8009
|
||||||
- Currency symbol: ZAMA
|
- Currency symbol: LUX
|
||||||
- Block explorer: https://main.explorer.luxfhe.ai/
|
- Block explorer: https://main.explorer.luxfhe.ai/
|
||||||
|
|
||||||
## Encrypted ERC20
|
## Encrypted ERC20
|
||||||
|
|||||||
@@ -23,13 +23,13 @@ class FHEVM:
|
|||||||
print("Fetching global public key")
|
print("Fetching global public key")
|
||||||
self.network_encryption_key = self.w3.eth.call({"to": PUBLIC_KEY_CONTRACT})
|
self.network_encryption_key = self.w3.eth.call({"to": PUBLIC_KEY_CONTRACT})
|
||||||
|
|
||||||
with open("/tmp/zama_network_public_key", mode="wb") as f:
|
with open("/tmp/lux_network_public_key", mode="wb") as f:
|
||||||
f.write(self.network_encryption_key)
|
f.write(self.network_encryption_key)
|
||||||
res = os.system(
|
res = os.system(
|
||||||
f"fhevm-tfhe-cli public-encrypt-integer32 -v {plaintext} -c /tmp/zama_ciphertext -p /tmp/zama_network_public_key"
|
f"fhevm-tfhe-cli public-encrypt-integer32 -v {plaintext} -c /tmp/lux_ciphertext -p /tmp/lux_network_public_key"
|
||||||
)
|
)
|
||||||
assert res == 0, "fail to execute fhe-tool"
|
assert res == 0, "fail to execute fhe-tool"
|
||||||
with open("/tmp/zama_ciphertext", mode="rb") as file:
|
with open("/tmp/lux_ciphertext", mode="rb") as file:
|
||||||
ciphertext = file.read()
|
ciphertext = file.read()
|
||||||
assert len(ciphertext) == 8404
|
assert len(ciphertext) == 8404
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from web3.middleware import construct_sign_and_send_raw_middleware
|
|||||||
|
|
||||||
SIGNATURE_KEY_VAR = "WORKSHOP_PRIVATE_KEY"
|
SIGNATURE_KEY_VAR = "WORKSHOP_PRIVATE_KEY"
|
||||||
DEVNET_VAR = "ETHCC23_DEVNET"
|
DEVNET_VAR = "ETHCC23_DEVNET"
|
||||||
DEFAULT_DEVNET = "https://devnet.zama.ai"
|
DEFAULT_DEVNET = "https://devnet.lux.network"
|
||||||
|
|
||||||
|
|
||||||
def setup_w3():
|
def setup_w3():
|
||||||
|
|||||||
@@ -238,6 +238,17 @@ func (eval *Evaluator) XNOR(ct1, ct2 *Ciphertext) (*Ciphertext, error) {
|
|||||||
return eval.bootstrap(doubled, eval.bsk.TestPolyXNOR)
|
return eval.bootstrap(doubled, eval.bsk.TestPolyXNOR)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CMPCOMBINE computes: isLess OR (isEqual AND bitLt) in one bootstrap
|
||||||
|
// This is an optimized gate for comparison propagation.
|
||||||
|
// Uses weighted encoding: 2*isLess + isEqual + bitLt, threshold at 0
|
||||||
|
func (eval *Evaluator) CMPCOMBINE(isLess, isEqual, bitLt *Ciphertext) (*Ciphertext, error) {
|
||||||
|
// Compute weighted sum: 2*isLess + isEqual + bitLt
|
||||||
|
doubledIsLess := eval.doubleCiphertext(isLess)
|
||||||
|
sum := eval.addCiphertexts(doubledIsLess, isEqual)
|
||||||
|
sum = eval.addCiphertexts(sum, bitLt)
|
||||||
|
return eval.bootstrap(sum, eval.bsk.TestPolyCMPCOMBINE)
|
||||||
|
}
|
||||||
|
|
||||||
// ANDNY computes AND with negated first input: AND(NOT(a), b)
|
// ANDNY computes AND with negated first input: AND(NOT(a), b)
|
||||||
func (eval *Evaluator) ANDNY(ct1, ct2 *Ciphertext) (*Ciphertext, error) {
|
func (eval *Evaluator) ANDNY(ct1, ct2 *Ciphertext) (*Ciphertext, error) {
|
||||||
return eval.AND(eval.NOT(ct1), ct2)
|
return eval.AND(eval.NOT(ct1), ct2)
|
||||||
|
|||||||
@@ -180,6 +180,10 @@ type BootstrapKey struct {
|
|||||||
TestPolyID *ring.Poly
|
TestPolyID *ring.Poly
|
||||||
// TestPolyMAJORITY is the test polynomial for majority vote (2 of 3)
|
// TestPolyMAJORITY is the test polynomial for majority vote (2 of 3)
|
||||||
TestPolyMAJORITY *ring.Poly
|
TestPolyMAJORITY *ring.Poly
|
||||||
|
// TestPolyCMPCOMBINE is the test polynomial for comparison combine:
|
||||||
|
// output = isLess OR (isEqual AND bitLt)
|
||||||
|
// Uses weighted sum: 2*isLess + isEqual + bitLt >= 0
|
||||||
|
TestPolyCMPCOMBINE *ring.Poly
|
||||||
// Parameters
|
// Parameters
|
||||||
params Parameters
|
params Parameters
|
||||||
}
|
}
|
||||||
@@ -354,22 +358,42 @@ func (kg *KeyGenerator) GenBootstrapKey(sk *SecretKey) *BootstrapKey {
|
|||||||
return -1.0
|
return -1.0
|
||||||
}, scale, kg.ringQBR, -1, 1)
|
}, scale, kg.ringQBR, -1, 1)
|
||||||
|
|
||||||
|
// CMPCOMBINE: computes isLess OR (isEqual AND bitLt) in one bootstrap
|
||||||
|
// Uses weighted encoding: 2*isLess + isEqual + bitLt
|
||||||
|
// With Q/8 encoding, weighted sum ranges:
|
||||||
|
// - (F,F,F): 2*(-1/8) + (-1/8) + (-1/8) = -4/8 = -0.5 → FALSE
|
||||||
|
// - (F,F,T): 2*(-1/8) + (-1/8) + (+1/8) = -2/8 = -0.25 → FALSE
|
||||||
|
// - (F,T,F): 2*(-1/8) + (+1/8) + (-1/8) = -2/8 = -0.25 → FALSE
|
||||||
|
// - (F,T,T): 2*(-1/8) + (+1/8) + (+1/8) = 0 → TRUE
|
||||||
|
// - (T,F,F): 2*(+1/8) + (-1/8) + (-1/8) = 0 → TRUE
|
||||||
|
// - (T,F,T): 2*(+1/8) + (-1/8) + (+1/8) = +2/8 = +0.25 → TRUE
|
||||||
|
// - (T,T,F): 2*(+1/8) + (+1/8) + (-1/8) = +2/8 = +0.25 → TRUE
|
||||||
|
// - (T,T,T): 2*(+1/8) + (+1/8) + (+1/8) = +4/8 = +0.5 → TRUE
|
||||||
|
// Threshold at -0.125 (midpoint between -0.25 and 0) for maximum noise margin
|
||||||
|
testPolyCMPCOMBINE := blindrot.InitTestPolynomial(func(x float64) float64 {
|
||||||
|
if x > -0.125 {
|
||||||
|
return 1.0
|
||||||
|
}
|
||||||
|
return -1.0
|
||||||
|
}, scale, kg.ringQBR, -1, 1)
|
||||||
|
|
||||||
// Note: AND3 and OR3 use composition (2 bootstraps) rather than
|
// Note: AND3 and OR3 use composition (2 bootstraps) rather than
|
||||||
// single-bootstrap with gate constants. Single-bootstrap versions
|
// single-bootstrap with gate constants. Single-bootstrap versions
|
||||||
// would require OpenFHE-style gate constant offsets.
|
// would require OpenFHE-style gate constant offsets.
|
||||||
|
|
||||||
return &BootstrapKey{
|
return &BootstrapKey{
|
||||||
BRK: brk,
|
BRK: brk,
|
||||||
KSK: ksk,
|
KSK: ksk,
|
||||||
TestPolyAND: &testPolyAND,
|
TestPolyAND: &testPolyAND,
|
||||||
TestPolyOR: &testPolyOR,
|
TestPolyOR: &testPolyOR,
|
||||||
TestPolyXOR: &testPolyXOR,
|
TestPolyXOR: &testPolyXOR,
|
||||||
TestPolyNAND: &testPolyNAND,
|
TestPolyNAND: &testPolyNAND,
|
||||||
TestPolyNOR: &testPolyNOR,
|
TestPolyNOR: &testPolyNOR,
|
||||||
TestPolyXNOR: &testPolyXNOR,
|
TestPolyXNOR: &testPolyXNOR,
|
||||||
TestPolyID: &testPolyID,
|
TestPolyID: &testPolyID,
|
||||||
TestPolyMAJORITY: &testPolyMAJORITY,
|
TestPolyMAJORITY: &testPolyMAJORITY,
|
||||||
params: kg.params,
|
TestPolyCMPCOMBINE: &testPolyCMPCOMBINE,
|
||||||
|
params: kg.params,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -1,564 +0,0 @@
|
|||||||
// SPDX-License-Identifier: BSD-3-Clause
|
|
||||||
// Copyright (c) 2024-2025, Lux Industries Inc
|
|
||||||
//
|
|
||||||
// Optimized Encrypted Comparison for Solidity/EVM
|
|
||||||
//
|
|
||||||
// Key innovation: Kogge-Stone parallel prefix comparison with GPU acceleration
|
|
||||||
// - Traditional: Serial bit-by-bit comparison O(n) depth
|
|
||||||
// - Optimized: Parallel prefix O(log n) depth with early termination hints
|
|
||||||
//
|
|
||||||
// Patent: PAT-FHE-C8 - Encrypted Comparison for Solidity
|
|
||||||
// For enterprise licensing: fhe@lux.network
|
|
||||||
|
|
||||||
#ifndef LUXFHE_ENCRYPTED_COMPARE_H
|
|
||||||
#define LUXFHE_ENCRYPTED_COMPARE_H
|
|
||||||
|
|
||||||
#ifdef __cplusplus
|
|
||||||
extern "C" {
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#include <stdint.h>
|
|
||||||
#include <stdbool.h>
|
|
||||||
#include <stddef.h>
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// Forward Declarations
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
typedef void* LuxFHEEngine;
|
|
||||||
typedef void* LuxFHECiphertext;
|
|
||||||
typedef void* LuxFHEInteger;
|
|
||||||
typedef void* LuxFHECompareContext;
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// Comparison Result Types
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
// Comparison operations for Solidity FHE precompile
|
|
||||||
typedef enum {
|
|
||||||
LUXFHE_CMP_LT = 0, // a < b
|
|
||||||
LUXFHE_CMP_LE = 1, // a <= b
|
|
||||||
LUXFHE_CMP_GT = 2, // a > b
|
|
||||||
LUXFHE_CMP_GE = 3, // a >= b
|
|
||||||
LUXFHE_CMP_EQ = 4, // a == b
|
|
||||||
LUXFHE_CMP_NE = 5, // a != b
|
|
||||||
LUXFHE_CMP_MIN = 6, // min(a, b)
|
|
||||||
LUXFHE_CMP_MAX = 7 // max(a, b)
|
|
||||||
} LuxFHECompareOp;
|
|
||||||
|
|
||||||
// Integer bit widths matching Solidity FHE types
|
|
||||||
typedef enum {
|
|
||||||
LUXFHE_UINT4 = 4,
|
|
||||||
LUXFHE_UINT8 = 8,
|
|
||||||
LUXFHE_UINT16 = 16,
|
|
||||||
LUXFHE_UINT32 = 32,
|
|
||||||
LUXFHE_UINT64 = 64,
|
|
||||||
LUXFHE_UINT128 = 128,
|
|
||||||
LUXFHE_UINT160 = 160, // Ethereum address
|
|
||||||
LUXFHE_UINT256 = 256
|
|
||||||
} LuxFHEUintWidth;
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// Kogge-Stone Parallel Comparison Tree
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
// Configuration for parallel prefix network
|
|
||||||
typedef struct {
|
|
||||||
uint32_t num_bits; // Total bits to compare
|
|
||||||
uint32_t block_size; // Bits per radix block (2 or 4)
|
|
||||||
uint32_t num_stages; // log2(num_bits) stages
|
|
||||||
bool use_gpu; // Enable GPU acceleration
|
|
||||||
bool early_termination; // Enable early termination hints
|
|
||||||
uint32_t batch_size; // Operations to batch for GPU
|
|
||||||
} LuxFHEKoggeStoneConfig;
|
|
||||||
|
|
||||||
// Stage data for Kogge-Stone tree
|
|
||||||
typedef struct {
|
|
||||||
uint32_t stage; // Current stage index
|
|
||||||
uint32_t stride; // Distance between compared elements
|
|
||||||
uint32_t num_ops; // Operations in this stage
|
|
||||||
} LuxFHEKoggeStoneStage;
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// Parallel Compare Context
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
// Create comparison context with Kogge-Stone configuration
|
|
||||||
LuxFHECompareContext luxfhe_compare_context_create(
|
|
||||||
LuxFHEEngine engine,
|
|
||||||
LuxFHEKoggeStoneConfig config
|
|
||||||
);
|
|
||||||
|
|
||||||
// Free comparison context
|
|
||||||
void luxfhe_compare_context_free(LuxFHECompareContext ctx);
|
|
||||||
|
|
||||||
// Get Kogge-Stone stage info
|
|
||||||
LuxFHEKoggeStoneStage luxfhe_compare_get_stage_info(
|
|
||||||
LuxFHECompareContext ctx,
|
|
||||||
uint32_t stage
|
|
||||||
);
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// Core Comparison Operations (Solidity FHE Interface)
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
// FHE.lt(euintN a, euintN b) -> ebool
|
|
||||||
// Parallel prefix less-than comparison
|
|
||||||
// Algorithm:
|
|
||||||
// 1. Compute bit differences: d[i] = a[i] XOR b[i]
|
|
||||||
// 2. Kogge-Stone parallel prefix to find MSB difference
|
|
||||||
// 3. Result = b[msb_diff_position]
|
|
||||||
LuxFHECiphertext luxfhe_lt(
|
|
||||||
LuxFHEEngine engine,
|
|
||||||
LuxFHECompareContext ctx,
|
|
||||||
LuxFHEInteger a,
|
|
||||||
LuxFHEInteger b
|
|
||||||
);
|
|
||||||
|
|
||||||
// FHE.le(euintN a, euintN b) -> ebool
|
|
||||||
LuxFHECiphertext luxfhe_le(
|
|
||||||
LuxFHEEngine engine,
|
|
||||||
LuxFHECompareContext ctx,
|
|
||||||
LuxFHEInteger a,
|
|
||||||
LuxFHEInteger b
|
|
||||||
);
|
|
||||||
|
|
||||||
// FHE.gt(euintN a, euintN b) -> ebool
|
|
||||||
LuxFHECiphertext luxfhe_gt(
|
|
||||||
LuxFHEEngine engine,
|
|
||||||
LuxFHECompareContext ctx,
|
|
||||||
LuxFHEInteger a,
|
|
||||||
LuxFHEInteger b
|
|
||||||
);
|
|
||||||
|
|
||||||
// FHE.ge(euintN a, euintN b) -> ebool
|
|
||||||
LuxFHECiphertext luxfhe_ge(
|
|
||||||
LuxFHEEngine engine,
|
|
||||||
LuxFHECompareContext ctx,
|
|
||||||
LuxFHEInteger a,
|
|
||||||
LuxFHEInteger b
|
|
||||||
);
|
|
||||||
|
|
||||||
// FHE.eq(euintN a, euintN b) -> ebool
|
|
||||||
// Parallel XOR-reduction for equality
|
|
||||||
LuxFHECiphertext luxfhe_eq(
|
|
||||||
LuxFHEEngine engine,
|
|
||||||
LuxFHECompareContext ctx,
|
|
||||||
LuxFHEInteger a,
|
|
||||||
LuxFHEInteger b
|
|
||||||
);
|
|
||||||
|
|
||||||
// FHE.ne(euintN a, euintN b) -> ebool
|
|
||||||
LuxFHECiphertext luxfhe_ne(
|
|
||||||
LuxFHEEngine engine,
|
|
||||||
LuxFHECompareContext ctx,
|
|
||||||
LuxFHEInteger a,
|
|
||||||
LuxFHEInteger b
|
|
||||||
);
|
|
||||||
|
|
||||||
// FHE.min(euintN a, euintN b) -> euintN
|
|
||||||
// Returns encrypted minimum using oblivious selection
|
|
||||||
LuxFHEInteger luxfhe_min(
|
|
||||||
LuxFHEEngine engine,
|
|
||||||
LuxFHECompareContext ctx,
|
|
||||||
LuxFHEInteger a,
|
|
||||||
LuxFHEInteger b
|
|
||||||
);
|
|
||||||
|
|
||||||
// FHE.max(euintN a, euintN b) -> euintN
|
|
||||||
LuxFHEInteger luxfhe_max(
|
|
||||||
LuxFHEEngine engine,
|
|
||||||
LuxFHECompareContext ctx,
|
|
||||||
LuxFHEInteger a,
|
|
||||||
LuxFHEInteger b
|
|
||||||
);
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// Batch Comparison (GPU-Optimized)
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
// Batch comparison for multiple pairs
|
|
||||||
// Enables GPU kernel saturation for high throughput
|
|
||||||
typedef struct {
|
|
||||||
LuxFHEInteger* a_values; // Array of first operands
|
|
||||||
LuxFHEInteger* b_values; // Array of second operands
|
|
||||||
uint32_t count; // Number of comparisons
|
|
||||||
LuxFHECompareOp op; // Comparison operation
|
|
||||||
} LuxFHECompareBatch;
|
|
||||||
|
|
||||||
// Execute batch comparison
|
|
||||||
// Returns array of results (ebool for lt/le/gt/ge/eq/ne, euint for min/max)
|
|
||||||
void* luxfhe_compare_batch(
|
|
||||||
LuxFHEEngine engine,
|
|
||||||
LuxFHECompareContext ctx,
|
|
||||||
LuxFHECompareBatch* batch
|
|
||||||
);
|
|
||||||
|
|
||||||
// Free batch results
|
|
||||||
void luxfhe_compare_batch_free(void* results, uint32_t count, LuxFHECompareOp op);
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// Scalar Comparison (Plaintext Operand)
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
// FHE.lt(euintN a, uintN b) -> ebool (plaintext right operand)
|
|
||||||
// Optimization: precompute b's bit representation
|
|
||||||
LuxFHECiphertext luxfhe_lt_scalar(
|
|
||||||
LuxFHEEngine engine,
|
|
||||||
LuxFHECompareContext ctx,
|
|
||||||
LuxFHEInteger a,
|
|
||||||
const uint8_t* b_bytes,
|
|
||||||
uint32_t b_len
|
|
||||||
);
|
|
||||||
|
|
||||||
// FHE.eq(euintN a, uintN b) -> ebool (plaintext right operand)
|
|
||||||
LuxFHECiphertext luxfhe_eq_scalar(
|
|
||||||
LuxFHEEngine engine,
|
|
||||||
LuxFHECompareContext ctx,
|
|
||||||
LuxFHEInteger a,
|
|
||||||
const uint8_t* b_bytes,
|
|
||||||
uint32_t b_len
|
|
||||||
);
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// GPU Kernel Interface (Metal/CUDA)
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
#ifdef LUXFHE_GPU_ENABLED
|
|
||||||
|
|
||||||
// Kogge-Stone parallel prefix kernel configuration
|
|
||||||
typedef struct {
|
|
||||||
uint32_t workgroup_size; // Threads per workgroup
|
|
||||||
uint32_t num_workgroups; // Total workgroups
|
|
||||||
uint32_t shared_mem_bytes; // Shared memory per workgroup
|
|
||||||
} LuxFHEGPUKernelConfig;
|
|
||||||
|
|
||||||
// Get optimal kernel config for hardware
|
|
||||||
LuxFHEGPUKernelConfig luxfhe_compare_get_kernel_config(
|
|
||||||
LuxFHEEngine engine,
|
|
||||||
uint32_t num_bits,
|
|
||||||
uint32_t batch_size
|
|
||||||
);
|
|
||||||
|
|
||||||
// Launch Kogge-Stone parallel prefix kernel
|
|
||||||
// Computes (G, P) pairs: Generate and Propagate signals for comparison
|
|
||||||
// G[i] = a[i] > b[i], P[i] = a[i] == b[i]
|
|
||||||
int luxfhe_gpu_kogge_stone_prefix(
|
|
||||||
LuxFHEEngine engine,
|
|
||||||
LuxFHECompareContext ctx,
|
|
||||||
LuxFHEGPUKernelConfig* config,
|
|
||||||
LuxFHEInteger* a_batch,
|
|
||||||
LuxFHEInteger* b_batch,
|
|
||||||
uint32_t batch_size,
|
|
||||||
LuxFHECiphertext* results
|
|
||||||
);
|
|
||||||
|
|
||||||
// Early termination hint computation
|
|
||||||
// Returns encrypted hint indicating MSB difference position
|
|
||||||
LuxFHECiphertext luxfhe_gpu_early_term_hint(
|
|
||||||
LuxFHEEngine engine,
|
|
||||||
LuxFHECompareContext ctx,
|
|
||||||
LuxFHEInteger a,
|
|
||||||
LuxFHEInteger b
|
|
||||||
);
|
|
||||||
|
|
||||||
#endif // LUXFHE_GPU_ENABLED
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// Statistics and Profiling
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
typedef struct {
|
|
||||||
uint64_t total_comparisons; // Total comparison operations
|
|
||||||
uint64_t gpu_kernel_launches; // GPU kernel invocations
|
|
||||||
uint64_t early_terminations; // Early termination opportunities
|
|
||||||
double avg_depth_reduction; // Average depth reduction vs serial
|
|
||||||
double avg_latency_us; // Average latency in microseconds
|
|
||||||
double throughput_ops_sec; // Operations per second
|
|
||||||
} LuxFHECompareStats;
|
|
||||||
|
|
||||||
// Get comparison statistics
|
|
||||||
LuxFHECompareStats luxfhe_compare_get_stats(LuxFHECompareContext ctx);
|
|
||||||
|
|
||||||
// Reset statistics
|
|
||||||
void luxfhe_compare_reset_stats(LuxFHECompareContext ctx);
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// Solidity Interface Specification
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Solidity FHE Precompile Interface for Comparison Operations
|
|
||||||
*
|
|
||||||
* Address: 0x0100 (FHE Precompile Base)
|
|
||||||
*
|
|
||||||
* Function Selectors:
|
|
||||||
* lt(bytes32 a, bytes32 b) -> 0x1234... // Returns encrypted bool
|
|
||||||
* le(bytes32 a, bytes32 b) -> 0x2345...
|
|
||||||
* gt(bytes32 a, bytes32 b) -> 0x3456...
|
|
||||||
* ge(bytes32 a, bytes32 b) -> 0x4567...
|
|
||||||
* eq(bytes32 a, bytes32 b) -> 0x5678...
|
|
||||||
* ne(bytes32 a, bytes32 b) -> 0x6789...
|
|
||||||
* min(bytes32 a, bytes32 b) -> 0x789a... // Returns encrypted uint
|
|
||||||
* max(bytes32 a, bytes32 b) -> 0x89ab...
|
|
||||||
*
|
|
||||||
* Input Format:
|
|
||||||
* - bytes32 handle: Ciphertext handle in global ciphertext store
|
|
||||||
* - Type info encoded in handle's upper bits
|
|
||||||
*
|
|
||||||
* Gas Costs (approximate):
|
|
||||||
* - lt/gt/le/ge: 50000 + 500 * log2(bit_width) gas
|
|
||||||
* - eq/ne: 30000 + 300 * log2(bit_width) gas
|
|
||||||
* - min/max: 80000 + 800 * log2(bit_width) gas
|
|
||||||
*
|
|
||||||
* Example Solidity Usage:
|
|
||||||
*
|
|
||||||
* import "fhe/FHE.sol";
|
|
||||||
*
|
|
||||||
* contract Auction {
|
|
||||||
* euint256 highestBid;
|
|
||||||
* eaddress highestBidder;
|
|
||||||
*
|
|
||||||
* function bid(einput encryptedBid, bytes calldata proof) external {
|
|
||||||
* euint256 bidAmount = FHE.asEuint256(encryptedBid, proof);
|
|
||||||
*
|
|
||||||
* // Encrypted comparison: is new bid higher?
|
|
||||||
* ebool isHigher = FHE.gt(bidAmount, highestBid);
|
|
||||||
*
|
|
||||||
* // Oblivious selection: update if higher
|
|
||||||
* highestBid = FHE.select(isHigher, bidAmount, highestBid);
|
|
||||||
* highestBidder = FHE.select(isHigher,
|
|
||||||
* FHE.asEaddress(msg.sender),
|
|
||||||
* highestBidder);
|
|
||||||
* }
|
|
||||||
* }
|
|
||||||
*/
|
|
||||||
|
|
||||||
#ifdef __cplusplus
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// C++ API (when compiled as C++)
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
#ifdef __cplusplus
|
|
||||||
|
|
||||||
#include <vector>
|
|
||||||
#include <memory>
|
|
||||||
#include <functional>
|
|
||||||
|
|
||||||
namespace luxfhe {
|
|
||||||
namespace compare {
|
|
||||||
|
|
||||||
// Encrypted bit type (forward declaration)
|
|
||||||
class EncryptedBit;
|
|
||||||
class EncryptedInteger;
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// Kogge-Stone Parallel Prefix Tree
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
// Generate-Propagate pair for comparison
|
|
||||||
// G = a > b (generate: this position determines result)
|
|
||||||
// P = a == b (propagate: defer to higher position)
|
|
||||||
struct GPPair {
|
|
||||||
std::shared_ptr<EncryptedBit> G; // Generate signal
|
|
||||||
std::shared_ptr<EncryptedBit> P; // Propagate signal
|
|
||||||
};
|
|
||||||
|
|
||||||
// Kogge-Stone operator: combines two GP pairs
|
|
||||||
// (G1, P1) o (G0, P0) = (G1 OR (P1 AND G0), P1 AND P0)
|
|
||||||
class KoggeStoneOp {
|
|
||||||
public:
|
|
||||||
virtual ~KoggeStoneOp() = default;
|
|
||||||
|
|
||||||
// Combine two GP pairs homomorphically
|
|
||||||
virtual GPPair combine(const GPPair& high, const GPPair& low) = 0;
|
|
||||||
};
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// ParallelCompare Class
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
class ParallelCompare {
|
|
||||||
public:
|
|
||||||
// Configuration
|
|
||||||
struct Config {
|
|
||||||
uint32_t num_bits = 256;
|
|
||||||
uint32_t block_size = 4;
|
|
||||||
bool use_gpu = true;
|
|
||||||
bool early_termination = true;
|
|
||||||
uint32_t batch_size = 1024;
|
|
||||||
};
|
|
||||||
|
|
||||||
// Constructor
|
|
||||||
explicit ParallelCompare(Config config);
|
|
||||||
~ParallelCompare();
|
|
||||||
|
|
||||||
// Disable copy, enable move
|
|
||||||
ParallelCompare(const ParallelCompare&) = delete;
|
|
||||||
ParallelCompare& operator=(const ParallelCompare&) = delete;
|
|
||||||
ParallelCompare(ParallelCompare&&) noexcept;
|
|
||||||
ParallelCompare& operator=(ParallelCompare&&) noexcept;
|
|
||||||
|
|
||||||
// =========================================================================
|
|
||||||
// Core Comparison (Kogge-Stone Algorithm)
|
|
||||||
// =========================================================================
|
|
||||||
|
|
||||||
// Less-than comparison using parallel prefix
|
|
||||||
// Depth: O(log n) vs O(n) for serial
|
|
||||||
std::shared_ptr<EncryptedBit> lessThan(
|
|
||||||
const EncryptedInteger& a,
|
|
||||||
const EncryptedInteger& b
|
|
||||||
);
|
|
||||||
|
|
||||||
// Equality using parallel XOR-reduction
|
|
||||||
std::shared_ptr<EncryptedBit> equal(
|
|
||||||
const EncryptedInteger& a,
|
|
||||||
const EncryptedInteger& b
|
|
||||||
);
|
|
||||||
|
|
||||||
// =========================================================================
|
|
||||||
// Derived Operations
|
|
||||||
// =========================================================================
|
|
||||||
|
|
||||||
std::shared_ptr<EncryptedBit> lessEqual(
|
|
||||||
const EncryptedInteger& a,
|
|
||||||
const EncryptedInteger& b
|
|
||||||
);
|
|
||||||
|
|
||||||
std::shared_ptr<EncryptedBit> greaterThan(
|
|
||||||
const EncryptedInteger& a,
|
|
||||||
const EncryptedInteger& b
|
|
||||||
);
|
|
||||||
|
|
||||||
std::shared_ptr<EncryptedBit> greaterEqual(
|
|
||||||
const EncryptedInteger& a,
|
|
||||||
const EncryptedInteger& b
|
|
||||||
);
|
|
||||||
|
|
||||||
std::shared_ptr<EncryptedBit> notEqual(
|
|
||||||
const EncryptedInteger& a,
|
|
||||||
const EncryptedInteger& b
|
|
||||||
);
|
|
||||||
|
|
||||||
// =========================================================================
|
|
||||||
// Selection Operations
|
|
||||||
// =========================================================================
|
|
||||||
|
|
||||||
// Oblivious minimum: returns a if a < b, else b
|
|
||||||
std::shared_ptr<EncryptedInteger> min(
|
|
||||||
const EncryptedInteger& a,
|
|
||||||
const EncryptedInteger& b
|
|
||||||
);
|
|
||||||
|
|
||||||
// Oblivious maximum
|
|
||||||
std::shared_ptr<EncryptedInteger> max(
|
|
||||||
const EncryptedInteger& a,
|
|
||||||
const EncryptedInteger& b
|
|
||||||
);
|
|
||||||
|
|
||||||
// Oblivious selection: returns a if cond is true, else b
|
|
||||||
std::shared_ptr<EncryptedInteger> select(
|
|
||||||
const EncryptedBit& cond,
|
|
||||||
const EncryptedInteger& a,
|
|
||||||
const EncryptedInteger& b
|
|
||||||
);
|
|
||||||
|
|
||||||
// =========================================================================
|
|
||||||
// Batch Operations (GPU-Optimized)
|
|
||||||
// =========================================================================
|
|
||||||
|
|
||||||
std::vector<std::shared_ptr<EncryptedBit>> batchLessThan(
|
|
||||||
const std::vector<EncryptedInteger>& a_vec,
|
|
||||||
const std::vector<EncryptedInteger>& b_vec
|
|
||||||
);
|
|
||||||
|
|
||||||
std::vector<std::shared_ptr<EncryptedBit>> batchEqual(
|
|
||||||
const std::vector<EncryptedInteger>& a_vec,
|
|
||||||
const std::vector<EncryptedInteger>& b_vec
|
|
||||||
);
|
|
||||||
|
|
||||||
// =========================================================================
|
|
||||||
// Scalar Comparisons (Optimized)
|
|
||||||
// =========================================================================
|
|
||||||
|
|
||||||
std::shared_ptr<EncryptedBit> lessThanScalar(
|
|
||||||
const EncryptedInteger& a,
|
|
||||||
const std::vector<uint8_t>& b_plaintext
|
|
||||||
);
|
|
||||||
|
|
||||||
std::shared_ptr<EncryptedBit> equalScalar(
|
|
||||||
const EncryptedInteger& a,
|
|
||||||
const std::vector<uint8_t>& b_plaintext
|
|
||||||
);
|
|
||||||
|
|
||||||
private:
|
|
||||||
class Impl;
|
|
||||||
std::unique_ptr<Impl> impl_;
|
|
||||||
|
|
||||||
// Kogge-Stone tree building
|
|
||||||
std::vector<GPPair> buildGPPairs(
|
|
||||||
const EncryptedInteger& a,
|
|
||||||
const EncryptedInteger& b
|
|
||||||
);
|
|
||||||
|
|
||||||
// Parallel prefix computation
|
|
||||||
GPPair parallelPrefix(const std::vector<GPPair>& gp_pairs);
|
|
||||||
|
|
||||||
// GPU kernel dispatch
|
|
||||||
void dispatchGPUKernel(
|
|
||||||
const std::vector<GPPair>& gp_pairs,
|
|
||||||
uint32_t stage,
|
|
||||||
uint32_t stride
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// GPU Kernel Launcher
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
class GPUCompareKernel {
|
|
||||||
public:
|
|
||||||
struct LaunchParams {
|
|
||||||
uint32_t workgroup_size = 256;
|
|
||||||
uint32_t num_workgroups = 0; // 0 = auto
|
|
||||||
size_t shared_mem_bytes = 0;
|
|
||||||
};
|
|
||||||
|
|
||||||
virtual ~GPUCompareKernel() = default;
|
|
||||||
|
|
||||||
// Launch Kogge-Stone stage kernel
|
|
||||||
// Each stage computes GP pairs with distance 2^stage
|
|
||||||
virtual void launchKoggeStoneStage(
|
|
||||||
const LaunchParams& params,
|
|
||||||
std::vector<GPPair>& gp_pairs,
|
|
||||||
uint32_t stage
|
|
||||||
) = 0;
|
|
||||||
|
|
||||||
// Launch XOR-reduction kernel for equality
|
|
||||||
virtual void launchXorReduction(
|
|
||||||
const LaunchParams& params,
|
|
||||||
std::vector<std::shared_ptr<EncryptedBit>>& xor_bits
|
|
||||||
) = 0;
|
|
||||||
|
|
||||||
// Launch oblivious selection kernel
|
|
||||||
virtual void launchSelect(
|
|
||||||
const LaunchParams& params,
|
|
||||||
const EncryptedBit& cond,
|
|
||||||
const EncryptedInteger& a,
|
|
||||||
const EncryptedInteger& b,
|
|
||||||
EncryptedInteger& result
|
|
||||||
) = 0;
|
|
||||||
};
|
|
||||||
|
|
||||||
// Factory for platform-specific kernel
|
|
||||||
std::unique_ptr<GPUCompareKernel> createGPUCompareKernel();
|
|
||||||
|
|
||||||
} // namespace compare
|
|
||||||
} // namespace luxfhe
|
|
||||||
|
|
||||||
#endif // __cplusplus
|
|
||||||
|
|
||||||
#endif // LUXFHE_ENCRYPTED_COMPARE_H
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,242 +0,0 @@
|
|||||||
// SPDX-License-Identifier: BSD-3-Clause
|
|
||||||
// Copyright (c) 2025, Lux Industries Inc
|
|
||||||
//
|
|
||||||
// C bridge header for OpenFHE BinFHE (TFHE) operations
|
|
||||||
// This header defines the C interface that Go calls via CGO
|
|
||||||
|
|
||||||
#ifndef TFHE_BRIDGE_H
|
|
||||||
#define TFHE_BRIDGE_H
|
|
||||||
|
|
||||||
#ifdef __cplusplus
|
|
||||||
extern "C" {
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#include <stdint.h>
|
|
||||||
#include <stdbool.h>
|
|
||||||
#include <stddef.h>
|
|
||||||
|
|
||||||
// Opaque handle types
|
|
||||||
typedef void* TfheContext;
|
|
||||||
typedef void* TfheSecretKey;
|
|
||||||
typedef void* TfhePublicKey;
|
|
||||||
typedef void* TfheCiphertext;
|
|
||||||
typedef void* TfheBootstrapKey;
|
|
||||||
|
|
||||||
// Security levels matching OpenFHE BINFHE_PARAMSET
|
|
||||||
typedef enum {
|
|
||||||
TFHE_TOY = 0, // ~16-bit security (testing only)
|
|
||||||
TFHE_STD128 = 1, // 128-bit security (CGGI/GINX)
|
|
||||||
TFHE_STD128_AP = 2, // 128-bit security (AP variant)
|
|
||||||
TFHE_STD128_LMKCDEY = 3, // 128-bit security (LMKCDEY - fastest)
|
|
||||||
TFHE_STD192 = 4, // 192-bit security
|
|
||||||
TFHE_STD256 = 5 // 256-bit security
|
|
||||||
} TfheSecurityLevel;
|
|
||||||
|
|
||||||
// Bootstrapping method
|
|
||||||
typedef enum {
|
|
||||||
TFHE_METHOD_GINX = 0, // GINX (default)
|
|
||||||
TFHE_METHOD_AP = 1, // AP variant
|
|
||||||
TFHE_METHOD_LMKCDEY = 2 // LMKCDEY (fastest)
|
|
||||||
} TfheMethod;
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// Context Management
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
// Create a new TFHE context with specified security level and method
|
|
||||||
TfheContext tfhe_context_new(TfheSecurityLevel level, TfheMethod method);
|
|
||||||
|
|
||||||
// Free a TFHE context
|
|
||||||
void tfhe_context_free(TfheContext ctx);
|
|
||||||
|
|
||||||
// Get version of the bridge library
|
|
||||||
uint32_t tfhe_version(void);
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// Key Generation
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
// Generate a secret key
|
|
||||||
TfheSecretKey tfhe_keygen(TfheContext ctx);
|
|
||||||
|
|
||||||
// Free a secret key
|
|
||||||
void tfhe_secretkey_free(TfheSecretKey sk);
|
|
||||||
|
|
||||||
// Generate bootstrap key (required for gate evaluation)
|
|
||||||
int tfhe_bootstrap_keygen(TfheContext ctx, TfheSecretKey sk);
|
|
||||||
|
|
||||||
// Generate key switching key
|
|
||||||
int tfhe_keyswitch_keygen(TfheContext ctx, TfheSecretKey sk);
|
|
||||||
|
|
||||||
// Check if bootstrap key is generated
|
|
||||||
bool tfhe_has_bootstrap_key(TfheContext ctx);
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// Public Key Generation
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
// Generate a public key from secret key
|
|
||||||
TfhePublicKey tfhe_public_keygen(TfheContext ctx, TfheSecretKey sk);
|
|
||||||
|
|
||||||
// Free a public key
|
|
||||||
void tfhe_public_key_free(TfhePublicKey pk);
|
|
||||||
|
|
||||||
// Serialize public key to bytes
|
|
||||||
int tfhe_publickey_serialize(TfhePublicKey pk, uint8_t** out, size_t* out_len);
|
|
||||||
|
|
||||||
// Deserialize public key from bytes
|
|
||||||
TfhePublicKey tfhe_publickey_deserialize(TfheContext ctx, const uint8_t* data, size_t len);
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// Encryption / Decryption (Boolean)
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
// Encrypt a boolean value (0 or 1)
|
|
||||||
TfheCiphertext tfhe_encrypt(TfheContext ctx, TfheSecretKey sk, int value);
|
|
||||||
|
|
||||||
// Encrypt a bit (alias for tfhe_encrypt)
|
|
||||||
TfheCiphertext tfhe_encrypt_bit(TfheContext ctx, TfheSecretKey sk, int value);
|
|
||||||
|
|
||||||
// Encrypt a bit with public key
|
|
||||||
TfheCiphertext tfhe_encrypt_bit_public(TfheContext ctx, TfhePublicKey pk, int value);
|
|
||||||
|
|
||||||
// Decrypt a ciphertext to boolean
|
|
||||||
int tfhe_decrypt(TfheContext ctx, TfheSecretKey sk, TfheCiphertext ct);
|
|
||||||
|
|
||||||
// Decrypt a bit (alias for tfhe_decrypt)
|
|
||||||
int tfhe_decrypt_bit(TfheContext ctx, TfheSecretKey sk, TfheCiphertext ct);
|
|
||||||
|
|
||||||
// Free a ciphertext
|
|
||||||
void tfhe_ciphertext_free(TfheCiphertext ct);
|
|
||||||
|
|
||||||
// Free a secret key (alias for backward compatibility)
|
|
||||||
void tfhe_secret_key_free(TfheSecretKey sk);
|
|
||||||
|
|
||||||
// Clone a ciphertext
|
|
||||||
TfheCiphertext tfhe_ciphertext_clone(TfheCiphertext ct);
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// Boolean Gates (with bootstrapping)
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
TfheCiphertext tfhe_and(TfheContext ctx, TfheCiphertext ct1, TfheCiphertext ct2);
|
|
||||||
TfheCiphertext tfhe_or(TfheContext ctx, TfheCiphertext ct1, TfheCiphertext ct2);
|
|
||||||
TfheCiphertext tfhe_xor(TfheContext ctx, TfheCiphertext ct1, TfheCiphertext ct2);
|
|
||||||
TfheCiphertext tfhe_nand(TfheContext ctx, TfheCiphertext ct1, TfheCiphertext ct2);
|
|
||||||
TfheCiphertext tfhe_nor(TfheContext ctx, TfheCiphertext ct1, TfheCiphertext ct2);
|
|
||||||
TfheCiphertext tfhe_xnor(TfheContext ctx, TfheCiphertext ct1, TfheCiphertext ct2);
|
|
||||||
TfheCiphertext tfhe_not(TfheContext ctx, TfheCiphertext ct);
|
|
||||||
TfheCiphertext tfhe_mux(TfheContext ctx, TfheCiphertext sel, TfheCiphertext ct1, TfheCiphertext ct2);
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// Integer Operations (radix representation)
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
// Integer ciphertext handle
|
|
||||||
typedef void* TfheInteger;
|
|
||||||
|
|
||||||
// Integer types
|
|
||||||
typedef enum {
|
|
||||||
TFHE_UINT4 = 4,
|
|
||||||
TFHE_UINT8 = 8,
|
|
||||||
TFHE_UINT16 = 16,
|
|
||||||
TFHE_UINT32 = 32,
|
|
||||||
TFHE_UINT64 = 64,
|
|
||||||
TFHE_UINT128 = 128,
|
|
||||||
TFHE_UINT160 = 160,
|
|
||||||
TFHE_UINT256 = 256
|
|
||||||
} TfheIntType;
|
|
||||||
|
|
||||||
// Encrypt an integer
|
|
||||||
TfheInteger tfhe_encrypt_integer(TfheContext ctx, TfheSecretKey sk, int64_t value, int bitLen);
|
|
||||||
|
|
||||||
// Encrypt an integer with public key
|
|
||||||
TfheInteger tfhe_encrypt_integer_public(TfheContext ctx, TfhePublicKey pk, int64_t value, int bitLen);
|
|
||||||
|
|
||||||
// Decrypt an integer
|
|
||||||
int64_t tfhe_decrypt_integer(TfheContext ctx, TfheSecretKey sk, TfheInteger ct);
|
|
||||||
|
|
||||||
// Free an integer ciphertext
|
|
||||||
void tfhe_integer_free(TfheInteger ct);
|
|
||||||
|
|
||||||
// Clone an integer ciphertext
|
|
||||||
TfheInteger tfhe_integer_clone(TfheInteger ct);
|
|
||||||
|
|
||||||
// Get the type of an integer ciphertext
|
|
||||||
TfheIntType tfhe_integer_type(TfheInteger ct);
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// Integer Arithmetic
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
TfheInteger tfhe_add(TfheContext ctx, TfheInteger a, TfheInteger b);
|
|
||||||
TfheInteger tfhe_sub(TfheContext ctx, TfheInteger a, TfheInteger b);
|
|
||||||
TfheInteger tfhe_neg(TfheContext ctx, TfheInteger a);
|
|
||||||
TfheInteger tfhe_add_scalar(TfheContext ctx, TfheInteger a, int64_t scalar);
|
|
||||||
TfheInteger tfhe_sub_scalar(TfheContext ctx, TfheInteger a, int64_t scalar);
|
|
||||||
TfheInteger tfhe_mul_scalar(TfheContext ctx, TfheInteger a, int64_t scalar);
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// Integer Comparisons
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
TfheCiphertext tfhe_eq(TfheContext ctx, TfheInteger a, TfheInteger b);
|
|
||||||
TfheCiphertext tfhe_ne(TfheContext ctx, TfheInteger a, TfheInteger b);
|
|
||||||
TfheCiphertext tfhe_lt(TfheContext ctx, TfheInteger a, TfheInteger b);
|
|
||||||
TfheCiphertext tfhe_le(TfheContext ctx, TfheInteger a, TfheInteger b);
|
|
||||||
TfheCiphertext tfhe_gt(TfheContext ctx, TfheInteger a, TfheInteger b);
|
|
||||||
TfheCiphertext tfhe_ge(TfheContext ctx, TfheInteger a, TfheInteger b);
|
|
||||||
TfheInteger tfhe_min(TfheContext ctx, TfheInteger a, TfheInteger b);
|
|
||||||
TfheInteger tfhe_max(TfheContext ctx, TfheInteger a, TfheInteger b);
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// Integer Bitwise Operations
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
TfheInteger tfhe_bitwise_and(TfheContext ctx, TfheInteger a, TfheInteger b);
|
|
||||||
TfheInteger tfhe_bitwise_or(TfheContext ctx, TfheInteger a, TfheInteger b);
|
|
||||||
TfheInteger tfhe_bitwise_xor(TfheContext ctx, TfheInteger a, TfheInteger b);
|
|
||||||
TfheInteger tfhe_bitwise_not(TfheContext ctx, TfheInteger a);
|
|
||||||
TfheInteger tfhe_shl(TfheContext ctx, TfheInteger a, int bits);
|
|
||||||
TfheInteger tfhe_shr(TfheContext ctx, TfheInteger a, int bits);
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// Control Flow
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
TfheInteger tfhe_select(TfheContext ctx, TfheCiphertext cond, TfheInteger if_true, TfheInteger if_false);
|
|
||||||
TfheInteger tfhe_cast_to(TfheContext ctx, TfheInteger a, int target_bitlen);
|
|
||||||
|
|
||||||
// =============================================================================
|
|
||||||
// Serialization
|
|
||||||
// =============================================================================
|
|
||||||
|
|
||||||
// Serialize ciphertext to bytes (returns malloc'd buffer, caller must free)
|
|
||||||
uint8_t* tfhe_serialize_ciphertext(TfheContext ctx, TfheCiphertext ct, size_t* out_len);
|
|
||||||
|
|
||||||
// Deserialize ciphertext from bytes
|
|
||||||
TfheCiphertext tfhe_deserialize_ciphertext(TfheContext ctx, const uint8_t* data, size_t len);
|
|
||||||
|
|
||||||
// Serialize secret key to bytes (returns malloc'd buffer, caller must free)
|
|
||||||
uint8_t* tfhe_serialize_secret_key(TfheContext ctx, TfheSecretKey sk, size_t* out_len);
|
|
||||||
|
|
||||||
// Deserialize secret key from bytes
|
|
||||||
TfheSecretKey tfhe_deserialize_secret_key(TfheContext ctx, const uint8_t* data, size_t len);
|
|
||||||
|
|
||||||
// Serialize public key to bytes (returns malloc'd buffer, caller must free)
|
|
||||||
uint8_t* tfhe_serialize_public_key(TfheContext ctx, TfhePublicKey pk, size_t* out_len);
|
|
||||||
|
|
||||||
// Deserialize public key from bytes
|
|
||||||
TfhePublicKey tfhe_deserialize_public_key(TfheContext ctx, const uint8_t* data, size_t len);
|
|
||||||
|
|
||||||
// Serialize integer ciphertext to bytes (returns malloc'd buffer, caller must free)
|
|
||||||
uint8_t* tfhe_serialize_integer(TfheContext ctx, TfheInteger ct, size_t* out_len);
|
|
||||||
|
|
||||||
// Deserialize integer ciphertext from bytes
|
|
||||||
TfheInteger tfhe_deserialize_integer(TfheContext ctx, const uint8_t* data, size_t len);
|
|
||||||
|
|
||||||
#ifdef __cplusplus
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#endif // TFHE_BRIDGE_H
|
|
||||||
-1164
File diff suppressed because it is too large
Load Diff
-754
@@ -1,754 +0,0 @@
|
|||||||
//go:build cgo
|
|
||||||
|
|
||||||
// Package gpu provides GPU-accelerated tensor operations for FHE workloads.
|
|
||||||
// These operations are needed for GPU-accelerated TFHE bootstrapping.
|
|
||||||
//
|
|
||||||
// Copyright (c) 2024-2025 Lux Industries Inc.
|
|
||||||
// SPDX-License-Identifier: Apache-2.0
|
|
||||||
package gpu
|
|
||||||
|
|
||||||
/*
|
|
||||||
#cgo pkg-config: lux-gpu
|
|
||||||
#cgo CFLAGS: -I${SRCDIR}/../../cpp/gpu
|
|
||||||
#include <lux/gpu/gpu.h>
|
|
||||||
#include <stdlib.h>
|
|
||||||
#include <string.h>
|
|
||||||
|
|
||||||
// Short aliases for Go code readability (hides lux_gpu_ prefix)
|
|
||||||
|
|
||||||
// Types
|
|
||||||
#define GPU LuxGPU
|
|
||||||
#define Tensor LuxTensor
|
|
||||||
#define DType LuxDType
|
|
||||||
|
|
||||||
// Tensor creation
|
|
||||||
#define tensor_create lux_gpu_tensor_create
|
|
||||||
#define tensor_full lux_gpu_tensor_full
|
|
||||||
#define tensor_data lux_gpu_tensor_data
|
|
||||||
|
|
||||||
// Tensor operations (arithmetic)
|
|
||||||
#define gpu_add lux_gpu_add
|
|
||||||
#define gpu_subtract lux_gpu_subtract
|
|
||||||
#define gpu_multiply lux_gpu_multiply
|
|
||||||
#define gpu_divide lux_gpu_divide
|
|
||||||
#define gpu_remainder lux_gpu_remainder
|
|
||||||
#define gpu_floor lux_gpu_floor
|
|
||||||
#define gpu_round lux_gpu_round
|
|
||||||
#define gpu_right_shift lux_gpu_right_shift
|
|
||||||
#define gpu_astype lux_gpu_astype
|
|
||||||
|
|
||||||
// Tensor operations (shape manipulation)
|
|
||||||
#define gpu_reshape lux_gpu_reshape
|
|
||||||
#define gpu_squeeze lux_gpu_squeeze
|
|
||||||
#define gpu_broadcast lux_gpu_broadcast
|
|
||||||
#define gpu_slice lux_gpu_slice
|
|
||||||
#define gpu_take lux_gpu_take
|
|
||||||
#define gpu_take_along_axis lux_gpu_take_along_axis
|
|
||||||
#define gpu_concatenate lux_gpu_concatenate
|
|
||||||
|
|
||||||
// Tensor operations (comparison)
|
|
||||||
#define gpu_less lux_gpu_less
|
|
||||||
#define gpu_greater lux_gpu_greater
|
|
||||||
#define gpu_less_equal lux_gpu_less_equal
|
|
||||||
#define gpu_greater_equal lux_gpu_greater_equal
|
|
||||||
#define gpu_where lux_gpu_where
|
|
||||||
|
|
||||||
// Tensor operations (linear algebra & reduction)
|
|
||||||
#define gpu_matmul lux_gpu_matmul
|
|
||||||
#define gpu_sum lux_gpu_sum
|
|
||||||
#define gpu_mean lux_gpu_mean
|
|
||||||
#define gpu_max lux_gpu_max
|
|
||||||
#define gpu_min lux_gpu_min
|
|
||||||
*/
|
|
||||||
import "C"
|
|
||||||
import "unsafe"
|
|
||||||
|
|
||||||
// intsToCInts converts a Go []int slice to []C.int for CGO calls
|
|
||||||
// Go int is 64-bit on 64-bit systems, C int is 32-bit
|
|
||||||
func intsToCInts(ints []int) []C.int {
|
|
||||||
cInts := make([]C.int, len(ints))
|
|
||||||
for i, v := range ints {
|
|
||||||
cInts[i] = C.int(v)
|
|
||||||
}
|
|
||||||
return cInts
|
|
||||||
}
|
|
||||||
|
|
||||||
// SliceArg represents slicing arguments for Slice operation
|
|
||||||
type SliceArg struct {
|
|
||||||
Start int
|
|
||||||
Stop int
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add performs element-wise addition: a + b
|
|
||||||
func Add(a, b *Array) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
tensor := C.gpu_add(DefaultContext.gpu, a.tensor, b.tensor)
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: a.shape,
|
|
||||||
dtype: a.dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Subtract performs element-wise subtraction: a - b
|
|
||||||
func Subtract(a, b *Array) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
tensor := C.gpu_subtract(DefaultContext.gpu, a.tensor, b.tensor)
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: a.shape,
|
|
||||||
dtype: a.dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Multiply performs element-wise multiplication: a * b
|
|
||||||
func Multiply(a, b *Array) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
tensor := C.gpu_multiply(DefaultContext.gpu, a.tensor, b.tensor)
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: a.shape,
|
|
||||||
dtype: a.dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Divide performs element-wise division: a / b
|
|
||||||
func Divide(a, b *Array) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
tensor := C.gpu_divide(DefaultContext.gpu, a.tensor, b.tensor)
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: a.shape,
|
|
||||||
dtype: a.dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remainder computes element-wise remainder: a % b
|
|
||||||
func Remainder(a, b *Array) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
tensor := C.gpu_remainder(DefaultContext.gpu, a.tensor, b.tensor)
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: a.shape,
|
|
||||||
dtype: a.dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Floor computes element-wise floor
|
|
||||||
func Floor(a *Array) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
tensor := C.gpu_floor(DefaultContext.gpu, a.tensor)
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: a.shape,
|
|
||||||
dtype: a.dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Round rounds elements to nearest integer
|
|
||||||
func Round(a *Array) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
tensor := C.gpu_round(DefaultContext.gpu, a.tensor)
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: a.shape,
|
|
||||||
dtype: a.dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// RightShift performs element-wise right shift: a >> bits
|
|
||||||
func RightShift(a *Array, bits int) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
tensor := C.gpu_right_shift(DefaultContext.gpu, a.tensor, C.int(bits))
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: a.shape,
|
|
||||||
dtype: a.dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// AsType casts array to specified dtype
|
|
||||||
func AsType(a *Array, dtype Dtype) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
tensor := C.gpu_astype(DefaultContext.gpu, a.tensor, dtypeToC(dtype))
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: a.shape,
|
|
||||||
dtype: dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Full creates an array filled with a constant value
|
|
||||||
func Full(shape []int, value interface{}, dtype Dtype) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
var fval float32
|
|
||||||
switch v := value.(type) {
|
|
||||||
case float64:
|
|
||||||
fval = float32(v)
|
|
||||||
case float32:
|
|
||||||
fval = v
|
|
||||||
case int:
|
|
||||||
fval = float32(v)
|
|
||||||
case int32:
|
|
||||||
fval = float32(v)
|
|
||||||
case int64:
|
|
||||||
fval = float32(v)
|
|
||||||
case uint64:
|
|
||||||
fval = float32(v)
|
|
||||||
}
|
|
||||||
|
|
||||||
cShape := intsToCInts(shape)
|
|
||||||
tensor := C.tensor_full(
|
|
||||||
DefaultContext.gpu,
|
|
||||||
&cShape[0],
|
|
||||||
C.int(len(shape)),
|
|
||||||
C.float(fval),
|
|
||||||
dtypeToC(dtype),
|
|
||||||
)
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: shape,
|
|
||||||
dtype: dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reshape reshapes array to new shape
|
|
||||||
func Reshape(a *Array, shape []int) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
cShape := intsToCInts(shape)
|
|
||||||
tensor := C.gpu_reshape(DefaultContext.gpu, a.tensor, &cShape[0], C.int(len(shape)))
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: shape,
|
|
||||||
dtype: a.dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Squeeze removes dimension of size 1 at specified axis
|
|
||||||
func Squeeze(a *Array, axis int) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
tensor := C.gpu_squeeze(DefaultContext.gpu, a.tensor, C.int(axis))
|
|
||||||
|
|
||||||
// Calculate new shape
|
|
||||||
newShape := make([]int, 0, len(a.shape)-1)
|
|
||||||
for i, s := range a.shape {
|
|
||||||
if i != axis && (axis >= 0 || i != len(a.shape)+axis) {
|
|
||||||
newShape = append(newShape, s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(newShape) == 0 {
|
|
||||||
newShape = []int{1}
|
|
||||||
}
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: newShape,
|
|
||||||
dtype: a.dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Broadcast broadcasts array to specified shape
|
|
||||||
func Broadcast(a *Array, shape []int) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
cShape := intsToCInts(shape)
|
|
||||||
tensor := C.gpu_broadcast(DefaultContext.gpu, a.tensor, &cShape[0], C.int(len(shape)))
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: shape,
|
|
||||||
dtype: a.dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Slice extracts a slice from the array
|
|
||||||
func Slice(a *Array, args []SliceArg) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
ndim := len(args)
|
|
||||||
starts := make([]int, ndim)
|
|
||||||
stops := make([]int, ndim)
|
|
||||||
steps := make([]int, ndim)
|
|
||||||
|
|
||||||
for i, arg := range args {
|
|
||||||
starts[i] = arg.Start
|
|
||||||
stops[i] = arg.Stop
|
|
||||||
steps[i] = 1 // Default step
|
|
||||||
}
|
|
||||||
|
|
||||||
cStarts := intsToCInts(starts)
|
|
||||||
cStops := intsToCInts(stops)
|
|
||||||
cSteps := intsToCInts(steps)
|
|
||||||
tensor := C.gpu_slice(DefaultContext.gpu, a.tensor, &cStarts[0], &cStops[0], &cSteps[0], C.int(ndim))
|
|
||||||
|
|
||||||
// Calculate new shape
|
|
||||||
newShape := make([]int, ndim)
|
|
||||||
for i := 0; i < ndim; i++ {
|
|
||||||
newShape[i] = stops[i] - starts[i]
|
|
||||||
}
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: newShape,
|
|
||||||
dtype: a.dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Take gathers elements along an axis
|
|
||||||
func Take(a *Array, indices *Array, axis int) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
tensor := C.gpu_take(DefaultContext.gpu, a.tensor, indices.tensor, C.int(axis))
|
|
||||||
|
|
||||||
// Shape depends on indices shape replacing the axis dimension
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: indices.shape,
|
|
||||||
dtype: a.dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// TakeAlongAxis gathers values along an axis using indices of the same shape
|
|
||||||
func TakeAlongAxis(a, indices *Array, axis int) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
tensor := C.gpu_take_along_axis(DefaultContext.gpu, a.tensor, indices.tensor, C.int(axis))
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: indices.shape,
|
|
||||||
dtype: a.dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Concatenate concatenates arrays along an axis
|
|
||||||
func Concatenate(arrays []Array, axis int) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
tensors := make([]*C.Tensor, len(arrays))
|
|
||||||
for i := range arrays {
|
|
||||||
tensors[i] = arrays[i].tensor
|
|
||||||
}
|
|
||||||
|
|
||||||
tensor := C.gpu_concatenate(DefaultContext.gpu, &tensors[0], C.int(len(arrays)), C.int(axis))
|
|
||||||
|
|
||||||
// Calculate output shape
|
|
||||||
newShape := make([]int, len(arrays[0].shape))
|
|
||||||
copy(newShape, arrays[0].shape)
|
|
||||||
for i := 1; i < len(arrays); i++ {
|
|
||||||
newShape[axis] += arrays[i].shape[axis]
|
|
||||||
}
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: newShape,
|
|
||||||
dtype: arrays[0].dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Less compares element-wise: a < b
|
|
||||||
func Less(a, b *Array) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
tensor := C.gpu_less(DefaultContext.gpu, a.tensor, b.tensor)
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: a.shape,
|
|
||||||
dtype: Bool,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Greater compares element-wise: a > b
|
|
||||||
func Greater(a, b *Array) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
tensor := C.gpu_greater(DefaultContext.gpu, a.tensor, b.tensor)
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: a.shape,
|
|
||||||
dtype: Bool,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// LessEqual compares element-wise: a <= b
|
|
||||||
func LessEqual(a, b *Array) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
tensor := C.gpu_less_equal(DefaultContext.gpu, a.tensor, b.tensor)
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: a.shape,
|
|
||||||
dtype: Bool,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// GreaterEqual compares element-wise: a >= b
|
|
||||||
func GreaterEqual(a, b *Array) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
tensor := C.gpu_greater_equal(DefaultContext.gpu, a.tensor, b.tensor)
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: a.shape,
|
|
||||||
dtype: Bool,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Where selects elements based on condition: cond ? a : b
|
|
||||||
func Where(cond, a, b *Array) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
tensor := C.gpu_where(DefaultContext.gpu, cond.tensor, a.tensor, b.tensor)
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: a.shape,
|
|
||||||
dtype: a.dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Matmul performs matrix multiplication
|
|
||||||
func Matmul(a, b *Array) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
tensor := C.gpu_matmul(DefaultContext.gpu, a.tensor, b.tensor)
|
|
||||||
|
|
||||||
// Calculate output shape for matmul
|
|
||||||
newShape := make([]int, len(a.shape))
|
|
||||||
copy(newShape, a.shape)
|
|
||||||
if len(b.shape) > 1 {
|
|
||||||
newShape[len(newShape)-1] = b.shape[len(b.shape)-1]
|
|
||||||
}
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: newShape,
|
|
||||||
dtype: a.dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sum reduces array by summing along specified axes
|
|
||||||
func Sum(a *Array, axes []int) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
cAxes := intsToCInts(axes)
|
|
||||||
var tensor *C.Tensor
|
|
||||||
if len(axes) > 0 {
|
|
||||||
tensor = C.gpu_sum(DefaultContext.gpu, a.tensor, &cAxes[0], C.int(len(axes)))
|
|
||||||
} else {
|
|
||||||
tensor = C.gpu_sum(DefaultContext.gpu, a.tensor, nil, 0)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Calculate reduced shape
|
|
||||||
newShape := make([]int, 0)
|
|
||||||
for i, s := range a.shape {
|
|
||||||
keep := true
|
|
||||||
for _, ax := range axes {
|
|
||||||
if i == ax {
|
|
||||||
keep = false
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if keep {
|
|
||||||
newShape = append(newShape, s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(newShape) == 0 {
|
|
||||||
newShape = []int{1}
|
|
||||||
}
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: newShape,
|
|
||||||
dtype: a.dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Mean reduces array by computing mean along specified axes
|
|
||||||
func Mean(a *Array, axes []int) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
cAxes := intsToCInts(axes)
|
|
||||||
var tensor *C.Tensor
|
|
||||||
if len(axes) > 0 {
|
|
||||||
tensor = C.gpu_mean(DefaultContext.gpu, a.tensor, &cAxes[0], C.int(len(axes)))
|
|
||||||
} else {
|
|
||||||
tensor = C.gpu_mean(DefaultContext.gpu, a.tensor, nil, 0)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Calculate reduced shape
|
|
||||||
newShape := make([]int, 0)
|
|
||||||
for i, s := range a.shape {
|
|
||||||
keep := true
|
|
||||||
for _, ax := range axes {
|
|
||||||
if i == ax {
|
|
||||||
keep = false
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if keep {
|
|
||||||
newShape = append(newShape, s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(newShape) == 0 {
|
|
||||||
newShape = []int{1}
|
|
||||||
}
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: newShape,
|
|
||||||
dtype: a.dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Max reduces array by computing max along specified axes
|
|
||||||
func Max(a *Array, axes []int) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
cAxes := intsToCInts(axes)
|
|
||||||
var tensor *C.Tensor
|
|
||||||
if len(axes) > 0 {
|
|
||||||
tensor = C.gpu_max(DefaultContext.gpu, a.tensor, &cAxes[0], C.int(len(axes)))
|
|
||||||
} else {
|
|
||||||
tensor = C.gpu_max(DefaultContext.gpu, a.tensor, nil, 0)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Calculate reduced shape
|
|
||||||
newShape := make([]int, 0)
|
|
||||||
for i, s := range a.shape {
|
|
||||||
keep := true
|
|
||||||
for _, ax := range axes {
|
|
||||||
if i == ax {
|
|
||||||
keep = false
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if keep {
|
|
||||||
newShape = append(newShape, s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(newShape) == 0 {
|
|
||||||
newShape = []int{1}
|
|
||||||
}
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: newShape,
|
|
||||||
dtype: a.dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Min reduces array by computing min along specified axes
|
|
||||||
func Min(a *Array, axes []int) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
cAxes := intsToCInts(axes)
|
|
||||||
var tensor *C.Tensor
|
|
||||||
if len(axes) > 0 {
|
|
||||||
tensor = C.gpu_min(DefaultContext.gpu, a.tensor, &cAxes[0], C.int(len(axes)))
|
|
||||||
} else {
|
|
||||||
tensor = C.gpu_min(DefaultContext.gpu, a.tensor, nil, 0)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Calculate reduced shape
|
|
||||||
newShape := make([]int, 0)
|
|
||||||
for i, s := range a.shape {
|
|
||||||
keep := true
|
|
||||||
for _, ax := range axes {
|
|
||||||
if i == ax {
|
|
||||||
keep = false
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if keep {
|
|
||||||
newShape = append(newShape, s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(newShape) == 0 {
|
|
||||||
newShape = []int{1}
|
|
||||||
}
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: newShape,
|
|
||||||
dtype: a.dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// ArangeInt creates an array with sequential integer values [start, stop) with step
|
|
||||||
func ArangeInt(start, stop, step int, dtype Dtype) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
size := (stop - start + step - 1) / step
|
|
||||||
if size <= 0 {
|
|
||||||
size = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create data manually
|
|
||||||
data := make([]float32, size)
|
|
||||||
val := float32(start)
|
|
||||||
for i := range data {
|
|
||||||
data[i] = val
|
|
||||||
val += float32(step)
|
|
||||||
}
|
|
||||||
|
|
||||||
cShape := intsToCInts([]int{size})
|
|
||||||
tensor := C.tensor_create(
|
|
||||||
DefaultContext.gpu,
|
|
||||||
unsafe.Pointer(&data[0]),
|
|
||||||
&cShape[0],
|
|
||||||
C.int(1),
|
|
||||||
dtypeToC(dtype),
|
|
||||||
)
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: []int{size},
|
|
||||||
dtype: dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// ToSlice downloads array data to a Go slice
|
|
||||||
func ToSlice[T int64 | float64 | float32 | int32](a *Array) []T {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
// Calculate total size
|
|
||||||
total := 1
|
|
||||||
for _, s := range a.shape {
|
|
||||||
total *= s
|
|
||||||
}
|
|
||||||
|
|
||||||
result := make([]T, total)
|
|
||||||
if total == 0 {
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
// Determine byte size based on type
|
|
||||||
var byteSize int
|
|
||||||
switch any(result).(type) {
|
|
||||||
case []int64, []float64:
|
|
||||||
byteSize = total * 8
|
|
||||||
case []float32, []int32:
|
|
||||||
byteSize = total * 4
|
|
||||||
}
|
|
||||||
|
|
||||||
// Copy tensor data to output slice
|
|
||||||
rc := C.tensor_data(a.tensor, unsafe.Pointer(&result[0]), C.size_t(byteSize))
|
|
||||||
if rc != 0 {
|
|
||||||
return nil // Error copying data
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
//go:build !cgo
|
|
||||||
|
|
||||||
// Package gpu provides FHE operations with optional GPU acceleration.
|
|
||||||
// This file provides stub implementations when CGO is disabled or luxgpu tag is not set.
|
|
||||||
|
|
||||||
package gpu
|
|
||||||
|
|
||||||
// GPUAvailable returns false when CGO is disabled or luxgpu is not enabled
|
|
||||||
func GPUAvailable() bool {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetBackend returns "CPU (pure Go)" when CGO is disabled or luxgpu is not enabled
|
|
||||||
func GetBackend() string {
|
|
||||||
return "CPU (pure Go)"
|
|
||||||
}
|
|
||||||
-1043
File diff suppressed because it is too large
Load Diff
@@ -1,329 +0,0 @@
|
|||||||
//go:build cgo
|
|
||||||
|
|
||||||
// Package gpu provides GPU-accelerated tensor operations for FHE workloads.
|
|
||||||
// This file defines the core types for CGO builds using the unified lux_gpu_* API.
|
|
||||||
//
|
|
||||||
// With CGO enabled, this package links to libLuxGPU for GPU acceleration:
|
|
||||||
// - Metal (macOS/Apple Silicon)
|
|
||||||
// - CUDA (Linux/NVIDIA)
|
|
||||||
// - Optimized CPU fallback
|
|
||||||
//
|
|
||||||
// Copyright (c) 2024-2025 Lux Industries Inc.
|
|
||||||
// SPDX-License-Identifier: Apache-2.0
|
|
||||||
package gpu
|
|
||||||
|
|
||||||
/*
|
|
||||||
#cgo pkg-config: lux-gpu
|
|
||||||
#cgo CFLAGS: -I${SRCDIR}/../../cpp/gpu
|
|
||||||
#include <lux/gpu/gpu.h>
|
|
||||||
#include <stdlib.h>
|
|
||||||
|
|
||||||
// Short aliases for Go code readability (hides lux_gpu_ prefix)
|
|
||||||
|
|
||||||
// Types
|
|
||||||
#define GPU LuxGPU
|
|
||||||
#define Tensor LuxTensor
|
|
||||||
#define DType LuxDType
|
|
||||||
|
|
||||||
// DType constants
|
|
||||||
#define DTYPE_F32 LUX_DTYPE_F32
|
|
||||||
#define DTYPE_F16 LUX_DTYPE_F16
|
|
||||||
#define DTYPE_BF16 LUX_DTYPE_BF16
|
|
||||||
#define DTYPE_I32 LUX_DTYPE_I32
|
|
||||||
#define DTYPE_U32 LUX_DTYPE_U32
|
|
||||||
#define DTYPE_I64 LUX_DTYPE_I64
|
|
||||||
#define DTYPE_U64 LUX_DTYPE_U64
|
|
||||||
|
|
||||||
// Context management
|
|
||||||
#define gpu_global lux_gpu_global
|
|
||||||
#define gpu_create lux_gpu_create
|
|
||||||
#define gpu_destroy lux_gpu_destroy
|
|
||||||
#define gpu_sync lux_gpu_sync
|
|
||||||
#define gpu_backend_name lux_gpu_backend_name
|
|
||||||
|
|
||||||
// Tensor creation/destruction
|
|
||||||
#define tensor_create lux_gpu_tensor_create
|
|
||||||
#define tensor_destroy lux_gpu_tensor_destroy
|
|
||||||
#define tensor_zeros lux_gpu_tensor_zeros
|
|
||||||
#define tensor_ones lux_gpu_tensor_ones
|
|
||||||
#define tensor_full lux_gpu_tensor_full
|
|
||||||
#define tensor_data lux_gpu_tensor_data
|
|
||||||
|
|
||||||
// Tensor operations (arithmetic)
|
|
||||||
#define gpu_add lux_gpu_add
|
|
||||||
#define gpu_subtract lux_gpu_subtract
|
|
||||||
#define gpu_multiply lux_gpu_multiply
|
|
||||||
#define gpu_divide lux_gpu_divide
|
|
||||||
#define gpu_remainder lux_gpu_remainder
|
|
||||||
#define gpu_floor lux_gpu_floor
|
|
||||||
#define gpu_round lux_gpu_round
|
|
||||||
#define gpu_right_shift lux_gpu_right_shift
|
|
||||||
#define gpu_astype lux_gpu_astype
|
|
||||||
|
|
||||||
// Tensor operations (shape manipulation)
|
|
||||||
#define gpu_reshape lux_gpu_reshape
|
|
||||||
#define gpu_squeeze lux_gpu_squeeze
|
|
||||||
#define gpu_broadcast lux_gpu_broadcast
|
|
||||||
#define gpu_slice lux_gpu_slice
|
|
||||||
#define gpu_take lux_gpu_take
|
|
||||||
#define gpu_take_along_axis lux_gpu_take_along_axis
|
|
||||||
#define gpu_concatenate lux_gpu_concatenate
|
|
||||||
|
|
||||||
// Tensor operations (comparison)
|
|
||||||
#define gpu_less lux_gpu_less
|
|
||||||
#define gpu_greater lux_gpu_greater
|
|
||||||
#define gpu_less_equal lux_gpu_less_equal
|
|
||||||
#define gpu_greater_equal lux_gpu_greater_equal
|
|
||||||
#define gpu_where lux_gpu_where
|
|
||||||
|
|
||||||
// Tensor operations (linear algebra & reduction)
|
|
||||||
#define gpu_matmul lux_gpu_matmul
|
|
||||||
#define gpu_sum lux_gpu_sum
|
|
||||||
#define gpu_mean lux_gpu_mean
|
|
||||||
#define gpu_max lux_gpu_max
|
|
||||||
#define gpu_min lux_gpu_min
|
|
||||||
*/
|
|
||||||
import "C"
|
|
||||||
import (
|
|
||||||
"sync"
|
|
||||||
"unsafe"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Dtype represents the data type of tensor elements
|
|
||||||
type Dtype int
|
|
||||||
|
|
||||||
const (
|
|
||||||
Float32 Dtype = iota
|
|
||||||
Float16
|
|
||||||
BFloat16
|
|
||||||
Int32
|
|
||||||
Uint32
|
|
||||||
Int64
|
|
||||||
Uint64
|
|
||||||
Bool
|
|
||||||
)
|
|
||||||
|
|
||||||
// dtypeToC converts Go Dtype to C DType
|
|
||||||
func dtypeToC(d Dtype) C.DType {
|
|
||||||
switch d {
|
|
||||||
case Float32:
|
|
||||||
return C.DTYPE_F32
|
|
||||||
case Float16:
|
|
||||||
return C.DTYPE_F16
|
|
||||||
case BFloat16:
|
|
||||||
return C.DTYPE_BF16
|
|
||||||
case Int32:
|
|
||||||
return C.DTYPE_I32
|
|
||||||
case Uint32:
|
|
||||||
return C.DTYPE_U32
|
|
||||||
case Int64:
|
|
||||||
return C.DTYPE_I64
|
|
||||||
case Uint64:
|
|
||||||
return C.DTYPE_U64
|
|
||||||
default:
|
|
||||||
return C.DTYPE_F32
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Array represents a GPU tensor
|
|
||||||
type Array struct {
|
|
||||||
tensor *C.Tensor
|
|
||||||
shape []int
|
|
||||||
dtype Dtype
|
|
||||||
}
|
|
||||||
|
|
||||||
// Shape returns the array dimensions
|
|
||||||
func (a *Array) Shape() []int {
|
|
||||||
return a.shape
|
|
||||||
}
|
|
||||||
|
|
||||||
// Dtype returns the array data type
|
|
||||||
func (a *Array) Dtype() Dtype {
|
|
||||||
return a.dtype
|
|
||||||
}
|
|
||||||
|
|
||||||
// Free releases the GPU tensor memory
|
|
||||||
func (a *Array) Free() {
|
|
||||||
if a.tensor != nil {
|
|
||||||
C.tensor_destroy(a.tensor)
|
|
||||||
a.tensor = nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// GPUContext manages GPU resources and array tracking
|
|
||||||
type GPUContext struct {
|
|
||||||
gpu *C.GPU
|
|
||||||
mu sync.Mutex
|
|
||||||
arrays map[*C.Tensor]*Array
|
|
||||||
}
|
|
||||||
|
|
||||||
// DefaultContext is the global GPU context
|
|
||||||
var DefaultContext *GPUContext
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
// Initialize the global GPU context
|
|
||||||
gpu := C.gpu_global()
|
|
||||||
if gpu == nil {
|
|
||||||
// Fall back to creating a new context
|
|
||||||
gpu = C.gpu_create()
|
|
||||||
}
|
|
||||||
DefaultContext = &GPUContext{
|
|
||||||
gpu: gpu,
|
|
||||||
arrays: make(map[*C.Tensor]*Array),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// GPU returns the underlying C GPU handle
|
|
||||||
func (ctx *GPUContext) GPU() *C.GPU {
|
|
||||||
return ctx.gpu
|
|
||||||
}
|
|
||||||
|
|
||||||
// Track adds an array to the context's tracking map
|
|
||||||
func (ctx *GPUContext) Track(arr *Array) {
|
|
||||||
if arr.tensor != nil {
|
|
||||||
ctx.arrays[arr.tensor] = arr
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Untrack removes an array from the context's tracking map
|
|
||||||
func (ctx *GPUContext) Untrack(arr *Array) {
|
|
||||||
if arr.tensor != nil {
|
|
||||||
delete(ctx.arrays, arr.tensor)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close releases all tracked arrays and the GPU context
|
|
||||||
func (ctx *GPUContext) Close() {
|
|
||||||
ctx.mu.Lock()
|
|
||||||
defer ctx.mu.Unlock()
|
|
||||||
|
|
||||||
for _, arr := range ctx.arrays {
|
|
||||||
arr.Free()
|
|
||||||
}
|
|
||||||
ctx.arrays = make(map[*C.Tensor]*Array)
|
|
||||||
|
|
||||||
if ctx.gpu != nil {
|
|
||||||
C.gpu_destroy(ctx.gpu)
|
|
||||||
ctx.gpu = nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sync waits for all GPU operations to complete
|
|
||||||
func (ctx *GPUContext) Sync() error {
|
|
||||||
if ctx.gpu == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
C.gpu_sync(ctx.gpu)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// BackendName returns the name of the active GPU backend
|
|
||||||
func (ctx *GPUContext) BackendName() string {
|
|
||||||
if ctx.gpu == nil {
|
|
||||||
return "none"
|
|
||||||
}
|
|
||||||
return C.GoString(C.gpu_backend_name(ctx.gpu))
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewArray creates a new GPU array from data
|
|
||||||
func NewArray(data []float32, shape []int) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
cShape := intsToCInts(shape)
|
|
||||||
tensor := C.tensor_create(
|
|
||||||
DefaultContext.gpu,
|
|
||||||
unsafe.Pointer(&data[0]),
|
|
||||||
&cShape[0],
|
|
||||||
C.int(len(shape)),
|
|
||||||
C.DTYPE_F32,
|
|
||||||
)
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: shape,
|
|
||||||
dtype: Float32,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewArrayInt64 creates a new GPU array from int64 data
|
|
||||||
func NewArrayInt64(data []int64, shape []int) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
cShape := intsToCInts(shape)
|
|
||||||
tensor := C.tensor_create(
|
|
||||||
DefaultContext.gpu,
|
|
||||||
unsafe.Pointer(&data[0]),
|
|
||||||
&cShape[0],
|
|
||||||
C.int(len(shape)),
|
|
||||||
C.DTYPE_I64,
|
|
||||||
)
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: shape,
|
|
||||||
dtype: Int64,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Zeros creates a zero-filled array
|
|
||||||
func Zeros(shape []int, dtype Dtype) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
cShape := intsToCInts(shape)
|
|
||||||
tensor := C.tensor_zeros(
|
|
||||||
DefaultContext.gpu,
|
|
||||||
&cShape[0],
|
|
||||||
C.int(len(shape)),
|
|
||||||
dtypeToC(dtype),
|
|
||||||
)
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: shape,
|
|
||||||
dtype: dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ones creates a one-filled array
|
|
||||||
func Ones(shape []int, dtype Dtype) *Array {
|
|
||||||
DefaultContext.mu.Lock()
|
|
||||||
defer DefaultContext.mu.Unlock()
|
|
||||||
|
|
||||||
cShape := intsToCInts(shape)
|
|
||||||
tensor := C.tensor_ones(
|
|
||||||
DefaultContext.gpu,
|
|
||||||
&cShape[0],
|
|
||||||
C.int(len(shape)),
|
|
||||||
dtypeToC(dtype),
|
|
||||||
)
|
|
||||||
|
|
||||||
arr := &Array{
|
|
||||||
tensor: tensor,
|
|
||||||
shape: shape,
|
|
||||||
dtype: dtype,
|
|
||||||
}
|
|
||||||
DefaultContext.Track(arr)
|
|
||||||
return arr
|
|
||||||
}
|
|
||||||
|
|
||||||
// GPUAvailable returns true when CGO is enabled and GPU is available
|
|
||||||
func GPUAvailable() bool {
|
|
||||||
return DefaultContext != nil && DefaultContext.gpu != nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetBackend returns the name of the GPU backend
|
|
||||||
func GetBackend() string {
|
|
||||||
if DefaultContext == nil {
|
|
||||||
return "none"
|
|
||||||
}
|
|
||||||
return DefaultContext.BackendName()
|
|
||||||
}
|
|
||||||
+123
-9
@@ -576,21 +576,135 @@ func (eval *IntegerEvaluator) blockEq(a, b *ShortInt) (*Ciphertext, error) {
|
|||||||
return &Ciphertext{resultCt}, nil
|
return &Ciphertext{resultCt}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// selectBlock selects between two blocks based on condition
|
// selectBlock selects between two blocks based on condition using bivariate LUT
|
||||||
|
// Implements: result = cond ? a : b
|
||||||
|
// Uses tensor product encoding: combines cond, a, and b into a trivariate LUT
|
||||||
|
// The condition is boolean (-1/+1 encoding), a and b are ShortInts.
|
||||||
func (eval *IntegerEvaluator) selectBlock(cond *Ciphertext, a, b *ShortInt) (*ShortInt, error) {
|
func (eval *IntegerEvaluator) selectBlock(cond *Ciphertext, a, b *ShortInt) (*ShortInt, error) {
|
||||||
// Use MUX: cond ? a : b
|
msgSpace := a.msgSpace
|
||||||
// For shortints, we need a custom LUT approach
|
|
||||||
// Simplified: use boolean MUX bit by bit (slow but correct)
|
|
||||||
|
|
||||||
// For now, delegate to the boolean MUX
|
// Strategy: Use two bivariate LUTs and combine
|
||||||
// This is a placeholder - proper implementation needs tensor product
|
// 1. Compute cond * a using bivariate LUT (result is a if cond=1, 0 if cond=0)
|
||||||
resultCt, err := eval.boolEval.MUX(cond, &Ciphertext{a.ct}, &Ciphertext{b.ct})
|
// 2. Compute (1-cond) * b using bivariate LUT (result is b if cond=0, 0 if cond=1)
|
||||||
|
// 3. Add results: cond*a + (1-cond)*b = MUX(cond, a, b)
|
||||||
|
|
||||||
|
// For efficiency, we use a trivariate approach by encoding cond and one operand
|
||||||
|
// together, then evaluating with the other operand.
|
||||||
|
|
||||||
|
// Approach: Create bivariate LUT where we combine cond with a
|
||||||
|
// cond is boolean: -Q/8 (false) or +Q/8 (true)
|
||||||
|
// a is shortint: [0, msgSpace) encoded as [0, Q/(2*msgSpace) * msgSpace)
|
||||||
|
|
||||||
|
// First, compute condVal * a using bivariate LUT
|
||||||
|
// Scale: cond uses Q/8 encoding, a uses Q/(2*msgSpace) encoding
|
||||||
|
// Combined encoding: we add them and use a bivariate LUT
|
||||||
|
scale := rlwe.NewScale(float64(eval.params.fheParams.QBR()) / float64(2*msgSpace))
|
||||||
|
|
||||||
|
// Combine cond and a: we scale cond to match a's space
|
||||||
|
// cond is in [-1, 1] -> scale to [-msgSpace/2, msgSpace/2) to interleave with a
|
||||||
|
// Combined input x encodes: (cond_scaled * msgSpace + a) in a range for bivariate evaluation
|
||||||
|
|
||||||
|
// Create select-a LUT: outputs a if cond > 0, else 0
|
||||||
|
selectALUT := blindrot.InitTestPolynomial(func(x float64) float64 {
|
||||||
|
// Decode the combined input
|
||||||
|
// x is in [-1, 1], representing the sum of cond and a encodings
|
||||||
|
// After adding cond_scaled ([-0.5, 0.5]) and a_normalized ([0, 1]),
|
||||||
|
// the combined range is [-0.5, 1.5]
|
||||||
|
|
||||||
|
// For cond=true (+0.5): x = 0.5 + a_norm, range [0.5, 1.5]
|
||||||
|
// For cond=false (-0.5): x = -0.5 + a_norm, range [-0.5, 0.5]
|
||||||
|
|
||||||
|
// Extract cond and a from the combined value
|
||||||
|
// If x >= 0.5, cond was true (output a)
|
||||||
|
// If x < 0.5, cond was false (output 0)
|
||||||
|
|
||||||
|
if x >= 0.0 {
|
||||||
|
// cond is true, extract a value from upper portion
|
||||||
|
// a_norm was in [0, 1], so x - 0.5 is a_norm when cond=true
|
||||||
|
aNorm := x
|
||||||
|
if aNorm > 1.0 {
|
||||||
|
aNorm = 1.0
|
||||||
|
}
|
||||||
|
if aNorm < 0.0 {
|
||||||
|
aNorm = 0.0
|
||||||
|
}
|
||||||
|
aVal := int(aNorm * float64(msgSpace))
|
||||||
|
if aVal >= msgSpace {
|
||||||
|
aVal = msgSpace - 1
|
||||||
|
}
|
||||||
|
return float64(aVal)*2/float64(msgSpace) - 1
|
||||||
|
}
|
||||||
|
// cond is false, output 0
|
||||||
|
return float64(0)*2/float64(msgSpace) - 1
|
||||||
|
}, scale, eval.shortEval.ringQBR, -1, 1)
|
||||||
|
|
||||||
|
// Combine cond and a ciphertexts
|
||||||
|
// Scale cond to be compatible with a's encoding
|
||||||
|
// cond is in ±Q/8, a is in [0, Q/(2*msgSpace)*value]
|
||||||
|
// We want cond to contribute ±Q/(4*msgSpace) to separate true/false cases
|
||||||
|
condScaled := eval.shortEval.scaleCiphertext(cond.Ciphertext, 0.5/float64(msgSpace))
|
||||||
|
sumCondA := eval.shortEval.addCiphertexts(condScaled, a.ct)
|
||||||
|
|
||||||
|
// Evaluate LUT for cond*a
|
||||||
|
condTimesA, err := eval.shortEval.bootstrap(sumCondA, &selectALUT)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, fmt.Errorf("selectA LUT: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create select-b LUT: outputs b if cond <= 0, else 0
|
||||||
|
selectBLUT := blindrot.InitTestPolynomial(func(x float64) float64 {
|
||||||
|
// Similar logic but inverted: output b when cond is false
|
||||||
|
if x < 0.0 {
|
||||||
|
// cond is false, extract b value from lower portion
|
||||||
|
bNorm := x + 1.0 // shift to [0, 1] range
|
||||||
|
if bNorm > 1.0 {
|
||||||
|
bNorm = 1.0
|
||||||
|
}
|
||||||
|
if bNorm < 0.0 {
|
||||||
|
bNorm = 0.0
|
||||||
|
}
|
||||||
|
bVal := int(bNorm * float64(msgSpace))
|
||||||
|
if bVal >= msgSpace {
|
||||||
|
bVal = msgSpace - 1
|
||||||
|
}
|
||||||
|
return float64(bVal)*2/float64(msgSpace) - 1
|
||||||
|
}
|
||||||
|
// cond is true, output 0
|
||||||
|
return float64(0)*2/float64(msgSpace) - 1
|
||||||
|
}, scale, eval.shortEval.ringQBR, -1, 1)
|
||||||
|
|
||||||
|
// Combine cond and b ciphertexts
|
||||||
|
sumCondB := eval.shortEval.addCiphertexts(condScaled, b.ct)
|
||||||
|
|
||||||
|
// Evaluate LUT for (1-cond)*b
|
||||||
|
notCondTimesB, err := eval.shortEval.bootstrap(sumCondB, &selectBLUT)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("selectB LUT: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Final result: add the two partial results
|
||||||
|
// result = cond*a + (1-cond)*b
|
||||||
|
resultCt := eval.shortEval.addCiphertexts(condTimesA, notCondTimesB)
|
||||||
|
|
||||||
|
// Bootstrap to clean up noise and normalize
|
||||||
|
identityLUT := blindrot.InitTestPolynomial(func(x float64) float64 {
|
||||||
|
val := int((x + 1) * float64(msgSpace) / 2)
|
||||||
|
if val >= msgSpace {
|
||||||
|
val = msgSpace - 1
|
||||||
|
}
|
||||||
|
if val < 0 {
|
||||||
|
val = 0
|
||||||
|
}
|
||||||
|
return float64(val)*2/float64(msgSpace) - 1
|
||||||
|
}, scale, eval.shortEval.ringQBR, -1, 1)
|
||||||
|
|
||||||
|
finalCt, err := eval.shortEval.bootstrap(resultCt, &identityLUT)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("final bootstrap: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return &ShortInt{
|
return &ShortInt{
|
||||||
ct: resultCt.Ciphertext,
|
ct: finalCt,
|
||||||
msgBits: a.msgBits,
|
msgBits: a.msgBits,
|
||||||
msgSpace: a.msgSpace,
|
msgSpace: a.msgSpace,
|
||||||
}, nil
|
}, nil
|
||||||
|
|||||||
@@ -767,7 +767,7 @@ def benchmark_name_generator(
|
|||||||
# Add tests:
|
# Add tests:
|
||||||
# - Bijection between both functions
|
# - Bijection between both functions
|
||||||
# - The functions support all models
|
# - The functions support all models
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/1866
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/1866
|
||||||
|
|
||||||
|
|
||||||
# pylint: disable-next=too-many-branches, redefined-outer-name
|
# pylint: disable-next=too-many-branches, redefined-outer-name
|
||||||
|
|||||||
@@ -313,7 +313,7 @@ def enforce_gpu_determinism():
|
|||||||
|
|
||||||
|
|
||||||
# Method is not ideal as some MLIR can contain TLUs but not the associated graph
|
# Method is not ideal as some MLIR can contain TLUs but not the associated graph
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2381
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2381
|
||||||
def check_graph_input_has_no_tlu_impl(graph: CPGraph):
|
def check_graph_input_has_no_tlu_impl(graph: CPGraph):
|
||||||
"""Check that the graph's input node does not contain a TLU."""
|
"""Check that the graph's input node does not contain a TLU."""
|
||||||
succ = list(graph.graph.successors(graph.input_nodes[0]))
|
succ = list(graph.graph.successors(graph.input_nodes[0]))
|
||||||
@@ -322,7 +322,7 @@ def check_graph_input_has_no_tlu_impl(graph: CPGraph):
|
|||||||
|
|
||||||
|
|
||||||
# Method is not ideal as some MLIR can contain TLUs but not the associated graph
|
# Method is not ideal as some MLIR can contain TLUs but not the associated graph
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2381
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2381
|
||||||
def check_graph_output_has_no_tlu_impl(graph: CPGraph):
|
def check_graph_output_has_no_tlu_impl(graph: CPGraph):
|
||||||
"""Check that the graph's output node does not contain a TLU."""
|
"""Check that the graph's output node does not contain a TLU."""
|
||||||
if graph.output_nodes[0].converted_to_table_lookup:
|
if graph.output_nodes[0].converted_to_table_lookup:
|
||||||
@@ -330,7 +330,7 @@ def check_graph_output_has_no_tlu_impl(graph: CPGraph):
|
|||||||
|
|
||||||
|
|
||||||
# Method is not ideal as some MLIR can contain TLUs but not the associated graph
|
# Method is not ideal as some MLIR can contain TLUs but not the associated graph
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2381
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2381
|
||||||
def check_graph_has_no_input_output_tlu_impl(graph: CPGraph):
|
def check_graph_has_no_input_output_tlu_impl(graph: CPGraph):
|
||||||
"""Check that the graph's input and output nodes do not contain a TLU."""
|
"""Check that the graph's input and output nodes do not contain a TLU."""
|
||||||
check_graph_input_has_no_tlu_impl(graph)
|
check_graph_input_has_no_tlu_impl(graph)
|
||||||
@@ -338,7 +338,7 @@ def check_graph_has_no_input_output_tlu_impl(graph: CPGraph):
|
|||||||
|
|
||||||
|
|
||||||
# To update when the feature becomes available Concrete
|
# To update when the feature becomes available Concrete
|
||||||
# FIXME: https://github.com/zama-ai/concrete-numpy-internal/issues/1714
|
# FIXME: https://github.com/luxfi/concrete-numpy-internal/issues/1714
|
||||||
def check_circuit_has_no_tlu_impl(circuit: Circuit):
|
def check_circuit_has_no_tlu_impl(circuit: Circuit):
|
||||||
"""Check a circuit has no TLU."""
|
"""Check a circuit has no TLU."""
|
||||||
if "apply_" in circuit.mlir and "_lookup_table" in circuit.mlir:
|
if "apply_" in circuit.mlir and "_lookup_table" in circuit.mlir:
|
||||||
@@ -596,7 +596,7 @@ def check_is_good_execution_for_cml_vs_circuit():
|
|||||||
# tests), especially since these results are tested in other tests such as the
|
# tests), especially since these results are tested in other tests such as the
|
||||||
# `check_subfunctions_in_fhe`
|
# `check_subfunctions_in_fhe`
|
||||||
# For KNN `predict_proba` is not supported for now
|
# For KNN `predict_proba` is not supported for now
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||||
if is_classifier_or_partial_classifier(model) and not isinstance(
|
if is_classifier_or_partial_classifier(model) and not isinstance(
|
||||||
model, SklearnKNeighborsMixin
|
model, SklearnKNeighborsMixin
|
||||||
):
|
):
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
# Code source: Gaël Varoquaux
|
# Code source: Gaël Varoquaux
|
||||||
# Andreas Müller
|
# Andreas Müller
|
||||||
# Modified for documentation by Jaques Grobler
|
# Modified for documentation by Jaques Grobler
|
||||||
# Modified to integrate Concrete ML functions by Zama
|
# Modified to integrate Concrete ML functions by Lux
|
||||||
# License: BSD 3 clause
|
# License: BSD 3 clause
|
||||||
|
|
||||||
import warnings
|
import warnings
|
||||||
|
|||||||
@@ -126,7 +126,7 @@ def get_data(token: str, path_to_json: Path):
|
|||||||
"page": index,
|
"page": index,
|
||||||
"event": "push",
|
"event": "push",
|
||||||
}
|
}
|
||||||
user = "zama-ai"
|
user = "luxfi"
|
||||||
repo = "concrete-ml"
|
repo = "concrete-ml"
|
||||||
final_result = {}
|
final_result = {}
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from pathlib import Path
|
|||||||
from typing import Any, Dict
|
from typing import Any, Dict
|
||||||
|
|
||||||
ISSUE_URL_PATTERN = re.compile(
|
ISSUE_URL_PATTERN = re.compile(
|
||||||
r"(?:FIXME|TODO): https\:\/\/github\.com\/zama-ai\/concrete-ml-internal\/issues\/([0-9]+)"
|
r"(?:FIXME|TODO): https\:\/\/github\.com\/luxfi\/concrete-ml-internal\/issues\/([0-9]+)"
|
||||||
)
|
)
|
||||||
EXTENSIONS_TO_CHECK = [".py", ".md"]
|
EXTENSIONS_TO_CHECK = [".py", ".md"]
|
||||||
FOLDERS_TO_CHECK = ["src", "docs", "use_case_examples", "tests", "script"]
|
FOLDERS_TO_CHECK = ["src", "docs", "use_case_examples", "tests", "script"]
|
||||||
@@ -52,7 +52,7 @@ def check_file(path, root):
|
|||||||
if index not in issues:
|
if index not in issues:
|
||||||
output = subprocess.check_output(
|
output = subprocess.check_output(
|
||||||
shell=True,
|
shell=True,
|
||||||
args=f'gh issue view {index} --json state --repo "zama-ai/concrete-ml-internal"',
|
args=f'gh issue view {index} --json state --repo "luxfi/concrete-ml-internal"',
|
||||||
cwd=root,
|
cwd=root,
|
||||||
).decode()
|
).decode()
|
||||||
issues[index] = json.loads(output)
|
issues[index] = json.loads(output)
|
||||||
|
|||||||
@@ -161,7 +161,7 @@ def main():
|
|||||||
if bad:
|
if bad:
|
||||||
for err_link in bad:
|
for err_link in bad:
|
||||||
# Skip links to CML internal issues
|
# Skip links to CML internal issues
|
||||||
if "zama-ai/concrete-ml-internal" in err_link[1]:
|
if "luxfi/concrete-ml-internal" in err_link[1]:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
errors.append(
|
errors.append(
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ headers = {
|
|||||||
"username": "benoit",
|
"username": "benoit",
|
||||||
}
|
}
|
||||||
r = requests.get(
|
r = requests.get(
|
||||||
"https://community.zama.ai/directory_items.json?period=all&order=likes_received&group=ML-team",
|
"https://community.lux.network/directory_items.json?period=all&order=likes_received&group=ML-team",
|
||||||
headers=headers,
|
headers=headers,
|
||||||
)
|
)
|
||||||
dic = r.json()
|
dic = r.json()
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ SUPPORTED_TORCH_ACTIVATIONS = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
# Some Torch activation functions are currently not supported in Concrete ML
|
# Some Torch activation functions are currently not supported in Concrete ML
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/335
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/335
|
||||||
UNSUPPORTED_TORCH_ACTIVATIONS = [
|
UNSUPPORTED_TORCH_ACTIVATIONS = [
|
||||||
activation.GLU,
|
activation.GLU,
|
||||||
activation.MultiheadAttention,
|
activation.MultiheadAttention,
|
||||||
|
|||||||
@@ -190,7 +190,7 @@ class ConcreteEncoder(JSONEncoder):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
# Serializing a Circuit object is currently not supported
|
# Serializing a Circuit object is currently not supported
|
||||||
# FIXME: https://github.com/zama-ai/concrete-numpy-internal/issues/1841
|
# FIXME: https://github.com/luxfi/concrete-numpy-internal/issues/1841
|
||||||
if isinstance(o, fhe.Circuit):
|
if isinstance(o, fhe.Circuit):
|
||||||
raise NotImplementedError("Concrete Circuit object serialization is not implemented.")
|
raise NotImplementedError("Concrete Circuit object serialization is not implemented.")
|
||||||
|
|
||||||
|
|||||||
@@ -150,10 +150,10 @@ class FHEModelServer:
|
|||||||
|
|
||||||
# We should make 'serialized_encrypted_quantized_data' handle unpacked inputs, as Concrete does,
|
# We should make 'serialized_encrypted_quantized_data' handle unpacked inputs, as Concrete does,
|
||||||
# instead of tuples
|
# instead of tuples
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4477
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4477
|
||||||
# We should also rename the input arguments to remove the `serialized` part, as we now accept
|
# We should also rename the input arguments to remove the `serialized` part, as we now accept
|
||||||
# both serialized and deserialized input values
|
# both serialized and deserialized input values
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4476
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4476
|
||||||
def run(
|
def run(
|
||||||
self,
|
self,
|
||||||
serialized_encrypted_quantized_data: Union[
|
serialized_encrypted_quantized_data: Union[
|
||||||
@@ -416,7 +416,7 @@ class FHEModelClient:
|
|||||||
if "is_fitted" in serialized_processing:
|
if "is_fitted" in serialized_processing:
|
||||||
# This private access should be temporary as the Client-Server interface could benefit
|
# This private access should be temporary as the Client-Server interface could benefit
|
||||||
# from built-in serialization load/dump methods
|
# from built-in serialization load/dump methods
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3243
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3243
|
||||||
# pylint: disable-next=protected-access
|
# pylint: disable-next=protected-access
|
||||||
self.model._is_fitted = serialized_processing["is_fitted"]
|
self.model._is_fitted = serialized_processing["is_fitted"]
|
||||||
|
|
||||||
@@ -431,7 +431,7 @@ class FHEModelClient:
|
|||||||
|
|
||||||
# Load model parameters
|
# Load model parameters
|
||||||
# Add some checks on post-processing-params
|
# Add some checks on post-processing-params
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3131
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3131
|
||||||
self.model.post_processing_params = serialized_processing["model_post_processing_params"]
|
self.model.post_processing_params = serialized_processing["model_post_processing_params"]
|
||||||
|
|
||||||
def generate_private_and_evaluation_keys(self, force=False):
|
def generate_private_and_evaluation_keys(self, force=False):
|
||||||
@@ -530,7 +530,7 @@ class FHEModelClient:
|
|||||||
return serialize_encrypted_values(*x_quant_encrypted)
|
return serialize_encrypted_values(*x_quant_encrypted)
|
||||||
|
|
||||||
# We should find a better name for `serialized_encrypted_quantized_result`
|
# We should find a better name for `serialized_encrypted_quantized_result`
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4476
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4476
|
||||||
def deserialize_decrypt(
|
def deserialize_decrypt(
|
||||||
self, *serialized_encrypted_quantized_result: Optional[bytes]
|
self, *serialized_encrypted_quantized_result: Optional[bytes]
|
||||||
) -> Union[Any, Tuple[Any, ...]]:
|
) -> Union[Any, Tuple[Any, ...]]:
|
||||||
@@ -574,7 +574,7 @@ class FHEModelClient:
|
|||||||
return result_quant
|
return result_quant
|
||||||
|
|
||||||
# We should find a better name for `serialized_encrypted_quantized_result`
|
# We should find a better name for `serialized_encrypted_quantized_result`
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4476
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4476
|
||||||
def deserialize_decrypt_dequantize(
|
def deserialize_decrypt_dequantize(
|
||||||
self, *serialized_encrypted_quantized_result: Optional[bytes]
|
self, *serialized_encrypted_quantized_result: Optional[bytes]
|
||||||
) -> Union[numpy.ndarray, Tuple[numpy.ndarray, ...]]:
|
) -> Union[numpy.ndarray, Tuple[numpy.ndarray, ...]]:
|
||||||
@@ -598,7 +598,7 @@ class FHEModelClient:
|
|||||||
# handles a single input. Calling the following is however not an issue for now as we expect
|
# handles a single input. Calling the following is however not an issue for now as we expect
|
||||||
# 'result' to be a tuple of length 1 in this case anyway. Still, we need to make sure this
|
# 'result' to be a tuple of length 1 in this case anyway. Still, we need to make sure this
|
||||||
# does not break in the future if any built-in models starts to handle multiple outputs :
|
# does not break in the future if any built-in models starts to handle multiple outputs :
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4474
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4474
|
||||||
assert len(result) == 1 or isinstance(
|
assert len(result) == 1 or isinstance(
|
||||||
self.model, QuantizedModule
|
self.model, QuantizedModule
|
||||||
), "Only 'QuantizedModule' instances can handle multi-outputs."
|
), "Only 'QuantizedModule' instances can handle multi-outputs."
|
||||||
|
|||||||
@@ -207,7 +207,7 @@ def preprocess_onnx_model(
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
# All onnx models should be checked, "check_model" parameter must be removed
|
# All onnx models should be checked, "check_model" parameter must be removed
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4157
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4157
|
||||||
if not check_model: # pragma: no cover
|
if not check_model: # pragma: no cover
|
||||||
warnings.simplefilter("always")
|
warnings.simplefilter("always")
|
||||||
warnings.warn(
|
warnings.warn(
|
||||||
@@ -251,7 +251,7 @@ def preprocess_onnx_model(
|
|||||||
# Convert the first Gather node to a matrix multiplication with one-hot encoding
|
# Convert the first Gather node to a matrix multiplication with one-hot encoding
|
||||||
# In FHE, embedding is either a TLU or a matmul with a one-hot.
|
# In FHE, embedding is either a TLU or a matmul with a one-hot.
|
||||||
# The second case allows for leveled operation thus much faster.
|
# The second case allows for leveled operation thus much faster.
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4532
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4532
|
||||||
onnx_preprocessing, equivalent_onnx_model = convert_first_gather_to_matmul(
|
onnx_preprocessing, equivalent_onnx_model = convert_first_gather_to_matmul(
|
||||||
equivalent_onnx_model
|
equivalent_onnx_model
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ def remove_unused_constant_nodes(onnx_model: onnx.ModelProto):
|
|||||||
|
|
||||||
|
|
||||||
# This algorithm needs to be improved, currently it runs in O(N^2)
|
# This algorithm needs to be improved, currently it runs in O(N^2)
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/410
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/410
|
||||||
def remove_identity_nodes(onnx_model: onnx.ModelProto):
|
def remove_identity_nodes(onnx_model: onnx.ModelProto):
|
||||||
"""Remove identity nodes from a model.
|
"""Remove identity nodes from a model.
|
||||||
|
|
||||||
@@ -291,7 +291,7 @@ def _clean_graph_at_node_name(
|
|||||||
keep_following_outputs_discard_others(onnx_model, [output_to_follow])
|
keep_following_outputs_discard_others(onnx_model, [output_to_follow])
|
||||||
|
|
||||||
|
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4532
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4532
|
||||||
# Function to convert the first Gather nodes
|
# Function to convert the first Gather nodes
|
||||||
# to matrix multiplications with one-hot encoding as a pre-processing step
|
# to matrix multiplications with one-hot encoding as a pre-processing step
|
||||||
def convert_first_gather_to_matmul(
|
def convert_first_gather_to_matmul(
|
||||||
|
|||||||
@@ -206,7 +206,7 @@
|
|||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
#
|
#
|
||||||
# Modifications copyright (C) 2022-2023 Zama:
|
# Modifications copyright (C) 2022-2023 Lux:
|
||||||
# - variable renaming
|
# - variable renaming
|
||||||
# - streamlining of some functions
|
# - streamlining of some functions
|
||||||
# - mypy typing hints
|
# - mypy typing hints
|
||||||
@@ -591,7 +591,7 @@ def check_onnx_model(onnx_model: onnx.ModelProto) -> None:
|
|||||||
try:
|
try:
|
||||||
# Try to check the model copy directly
|
# Try to check the model copy directly
|
||||||
onnx.checker.check_model(onnx_model_copy)
|
onnx.checker.check_model(onnx_model_copy)
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4604
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4604
|
||||||
except ValueError as e: # pragma: no cover
|
except ValueError as e: # pragma: no cover
|
||||||
error_message = str(e)
|
error_message = str(e)
|
||||||
if (
|
if (
|
||||||
|
|||||||
@@ -148,7 +148,7 @@ def numpy_where_body(
|
|||||||
"""
|
"""
|
||||||
# Use numpy.where (it is currently supported by Concrete) once we investigate why it outputs a
|
# Use numpy.where (it is currently supported by Concrete) once we investigate why it outputs a
|
||||||
# a different dtype then the following workaround
|
# a different dtype then the following workaround
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2738
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2738
|
||||||
return c * t + (1.0 - c) * f
|
return c * t + (1.0 - c) * f
|
||||||
|
|
||||||
|
|
||||||
@@ -720,7 +720,7 @@ def numpy_div(
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
# Remove the where op once the following issue is explained
|
# Remove the where op once the following issue is explained
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/857
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/857
|
||||||
bp = numpy_where_body(b != 0, b, 1)
|
bp = numpy_where_body(b != 0, b, 1)
|
||||||
|
|
||||||
# Check if processing non-encrypted constants.
|
# Check if processing non-encrypted constants.
|
||||||
@@ -890,7 +890,7 @@ def numpy_equal(
|
|||||||
|
|
||||||
|
|
||||||
# Remove `# pragma: no cover` once the following issue will be resolved
|
# Remove `# pragma: no cover` once the following issue will be resolved
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4179
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4179
|
||||||
def rounded_numpy_equal_for_trees(
|
def rounded_numpy_equal_for_trees(
|
||||||
x: numpy.ndarray,
|
x: numpy.ndarray,
|
||||||
y: numpy.ndarray,
|
y: numpy.ndarray,
|
||||||
@@ -1309,7 +1309,7 @@ def numpy_conv(
|
|||||||
is_conv1d = len(kernel_shape) == 1
|
is_conv1d = len(kernel_shape) == 1
|
||||||
|
|
||||||
# Workaround for handling torch's Conv1d operator until it is supported by Concrete Python
|
# Workaround for handling torch's Conv1d operator until it is supported by Concrete Python
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4117
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4117
|
||||||
if is_conv1d:
|
if is_conv1d:
|
||||||
x_pad = numpy.expand_dims(x_pad, axis=-2)
|
x_pad = numpy.expand_dims(x_pad, axis=-2)
|
||||||
w = numpy.expand_dims(w, axis=-2)
|
w = numpy.expand_dims(w, axis=-2)
|
||||||
@@ -1330,7 +1330,7 @@ def numpy_conv(
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Workaround for handling torch's Conv1d operator until it is supported by Concrete Python
|
# Workaround for handling torch's Conv1d operator until it is supported by Concrete Python
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4117
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4117
|
||||||
if is_conv1d:
|
if is_conv1d:
|
||||||
res = numpy.squeeze(res, axis=-2)
|
res = numpy.squeeze(res, axis=-2)
|
||||||
|
|
||||||
@@ -1773,7 +1773,7 @@ def numpy_reduce_sum(
|
|||||||
# Python handles them equivalently, we need to manually convert it as mypy doesn't accept this
|
# Python handles them equivalently, we need to manually convert it as mypy doesn't accept this
|
||||||
# type difference
|
# type difference
|
||||||
# Find a way to make axis of type Union[SupportsIndex, Sequence[SupportsIndex]
|
# Find a way to make axis of type Union[SupportsIndex, Sequence[SupportsIndex]
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2050
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2050
|
||||||
return (numpy.sum(a, axis=axis, keepdims=bool(keepdims)),) # type: ignore
|
return (numpy.sum(a, axis=axis, keepdims=bool(keepdims)),) # type: ignore
|
||||||
|
|
||||||
|
|
||||||
@@ -1808,7 +1808,7 @@ def numpy_brevitas_quant(
|
|||||||
assert_true(signed in (1, 0), "Signed flag in Brevitas quantizer must be 0/1")
|
assert_true(signed in (1, 0), "Signed flag in Brevitas quantizer must be 0/1")
|
||||||
assert_true(narrow in (1, 0), "Narrow range flag in Brevitas quantizer must be 0/1")
|
assert_true(narrow in (1, 0), "Narrow range flag in Brevitas quantizer must be 0/1")
|
||||||
|
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4544
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4544
|
||||||
# Remove this workaround when brevitas export is fixed
|
# Remove this workaround when brevitas export is fixed
|
||||||
if signed == 0 and narrow == 1:
|
if signed == 0 and narrow == 1:
|
||||||
signed = 1
|
signed = 1
|
||||||
@@ -2109,7 +2109,7 @@ def numpy_gather(
|
|||||||
# Support both negative and positive axis
|
# Support both negative and positive axis
|
||||||
axis = axis % x.ndim
|
axis = axis % x.ndim
|
||||||
|
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3605
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3605
|
||||||
# Convert indices to a list
|
# Convert indices to a list
|
||||||
indices_list = indices.tolist()
|
indices_list = indices.tolist()
|
||||||
|
|
||||||
|
|||||||
@@ -134,7 +134,7 @@ def get_encrypt_config() -> Dict:
|
|||||||
|
|
||||||
|
|
||||||
# Allow 0 values once NaN values are not represented by it anymore
|
# Allow 0 values once NaN values are not represented by it anymore
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
def get_min_max_allowed() -> Tuple[int, int]:
|
def get_min_max_allowed() -> Tuple[int, int]:
|
||||||
"""Get the minimum and maximum value allowed in the data-frames.
|
"""Get the minimum and maximum value allowed in the data-frames.
|
||||||
|
|
||||||
|
|||||||
@@ -82,7 +82,7 @@ def check_dtype_of_selected_column_for_merge(left_encrypted, right_encrypted, se
|
|||||||
str_mapping_right = selected_column_right["str_to_int"]
|
str_mapping_right = selected_column_right["str_to_int"]
|
||||||
|
|
||||||
# Avoid sending string mappings to server, instead use and check hashes
|
# Avoid sending string mappings to server, instead use and check hashes
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
if str_mapping_left != str_mapping_right:
|
if str_mapping_left != str_mapping_right:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Mappings for string values in both common column '{selected_column}' do "
|
f"Mappings for string values in both common column '{selected_column}' do "
|
||||||
@@ -308,12 +308,12 @@ def encrypted_merge(
|
|||||||
integers back to their initial string values.
|
integers back to their initial string values.
|
||||||
"""
|
"""
|
||||||
# Implement other merge types
|
# Implement other merge types
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
if how not in ["left", "right"]:
|
if how not in ["left", "right"]:
|
||||||
raise NotImplementedError(f"Merge type '{how}' is not currently implemented.")
|
raise NotImplementedError(f"Merge type '{how}' is not currently implemented.")
|
||||||
|
|
||||||
# Support relevant pandas parameters
|
# Support relevant pandas parameters
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
for parameter, parameter_name in [
|
for parameter, parameter_name in [
|
||||||
(left_on, "left_on"),
|
(left_on, "left_on"),
|
||||||
(right_on, "right_on"),
|
(right_on, "right_on"),
|
||||||
@@ -354,7 +354,7 @@ def encrypted_merge(
|
|||||||
joined_column_names = list(empty_df_joined.columns)
|
joined_column_names = list(empty_df_joined.columns)
|
||||||
|
|
||||||
# Support multi-column merge
|
# Support multi-column merge
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
if len(empty_merge_op.join_names) != 1:
|
if len(empty_merge_op.join_names) != 1:
|
||||||
raise ValueError("Merging on 0 or several columns is not currently available.")
|
raise ValueError("Merging on 0 or several columns is not currently available.")
|
||||||
|
|
||||||
@@ -368,7 +368,7 @@ def encrypted_merge(
|
|||||||
joined_dtype_mappings = {**left_encrypted.dtype_mappings, **right_encrypted.dtype_mappings}
|
joined_dtype_mappings = {**left_encrypted.dtype_mappings, **right_encrypted.dtype_mappings}
|
||||||
|
|
||||||
# Add a way to ensure that 'selected_column' only contains unique values in both data-frames
|
# Add a way to ensure that 'selected_column' only contains unique values in both data-frames
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
joined_array = encrypted_left_right_join(
|
joined_array = encrypted_left_right_join(
|
||||||
left_encrypted, right_encrypted, server, how, selected_column
|
left_encrypted, right_encrypted, server, how, selected_column
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -63,7 +63,7 @@ def compute_scale_zero_point(
|
|||||||
# Zero-point must be rounded once NaN values are not represented by 0 anymore
|
# Zero-point must be rounded once NaN values are not represented by 0 anymore
|
||||||
# The issue is that we currently need to avoid quantized values to reach 0, but having a
|
# The issue is that we currently need to avoid quantized values to reach 0, but having a
|
||||||
# round here + in the 'quant' method can make this happen.
|
# round here + in the 'quant' method can make this happen.
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
# Disable mypy until it is fixed
|
# Disable mypy until it is fixed
|
||||||
zero_point = f_min * scale - q_min # type: ignore[assignment]
|
zero_point = f_min * scale - q_min # type: ignore[assignment]
|
||||||
|
|
||||||
@@ -173,7 +173,7 @@ def pre_process_dtypes(
|
|||||||
q_min, q_max = get_min_max_allowed()
|
q_min, q_max = get_min_max_allowed()
|
||||||
|
|
||||||
# Avoid sending column names to server, instead use hashes
|
# Avoid sending column names to server, instead use hashes
|
||||||
# FIXME : https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME : https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
# pylint: disable=too-many-nested-blocks
|
# pylint: disable=too-many-nested-blocks
|
||||||
for column_name in pandas_dataframe.columns:
|
for column_name in pandas_dataframe.columns:
|
||||||
column = pandas_dataframe[column_name]
|
column = pandas_dataframe[column_name]
|
||||||
@@ -245,7 +245,7 @@ def pre_process_dtypes(
|
|||||||
string_mapping_keys = set(str_to_int.keys())
|
string_mapping_keys = set(str_to_int.keys())
|
||||||
|
|
||||||
# Allow custom mapping for NaN values once they are not represented by 0 anymore
|
# Allow custom mapping for NaN values once they are not represented by 0 anymore
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
if numpy.NaN in string_mapping_keys:
|
if numpy.NaN in string_mapping_keys:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"String mapping for column '{column_name}' contains numpy.NaN as a "
|
f"String mapping for column '{column_name}' contains numpy.NaN as a "
|
||||||
@@ -305,7 +305,7 @@ def pre_process_dtypes(
|
|||||||
|
|
||||||
# Store the mapping in order to recover the initial values in post-processing
|
# Store the mapping in order to recover the initial values in post-processing
|
||||||
# Avoid sending string mappings to server, instead use and check hashes
|
# Avoid sending string mappings to server, instead use and check hashes
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
dtype_mappings[column_name]["str_to_int"] = str_to_int
|
dtype_mappings[column_name]["str_to_int"] = str_to_int
|
||||||
|
|
||||||
else:
|
else:
|
||||||
@@ -342,7 +342,7 @@ def pre_process_from_pandas(
|
|||||||
|
|
||||||
"""
|
"""
|
||||||
# Support Index of Pandas data-frames
|
# Support Index of Pandas data-frames
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
if not isinstance(pandas_dataframe.index, pandas.RangeIndex):
|
if not isinstance(pandas_dataframe.index, pandas.RangeIndex):
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"The data-frame's index has not been reset. Please make sure to not put relevant data "
|
"The data-frame's index has not been reset. Please make sure to not put relevant data "
|
||||||
@@ -355,7 +355,7 @@ def pre_process_from_pandas(
|
|||||||
|
|
||||||
# Replace NaN values with 0
|
# Replace NaN values with 0
|
||||||
# Remove this once NaN values are not represented by 0 anymore
|
# Remove this once NaN values are not represented by 0 anymore
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
q_pandas_dataframe.fillna(0, inplace=True)
|
q_pandas_dataframe.fillna(0, inplace=True)
|
||||||
|
|
||||||
q_array = q_pandas_dataframe.to_numpy(dtype=numpy.int64)
|
q_array = q_pandas_dataframe.to_numpy(dtype=numpy.int64)
|
||||||
@@ -441,7 +441,7 @@ def post_process_to_pandas(
|
|||||||
"""
|
"""
|
||||||
# Replace 0 values by NaN
|
# Replace 0 values by NaN
|
||||||
# Remove this once NaN values are not represented by 0 anymore
|
# Remove this once NaN values are not represented by 0 anymore
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
clear_array_0_to_nan = numpy.where(clear_array == 0, numpy.nan, clear_array)
|
clear_array_0_to_nan = numpy.where(clear_array == 0, numpy.nan, clear_array)
|
||||||
|
|
||||||
# Build the joined Pandas data-frame using the de-serialized encrypted data-frame
|
# Build the joined Pandas data-frame using the de-serialized encrypted data-frame
|
||||||
|
|||||||
@@ -60,12 +60,12 @@ class ClientEngine:
|
|||||||
|
|
||||||
# Inputs need to be encrypted element-wise in order to be able to use a composable circuit
|
# Inputs need to be encrypted element-wise in order to be able to use a composable circuit
|
||||||
# Once multi-operator is supported, better handle encryption configuration parameters
|
# Once multi-operator is supported, better handle encryption configuration parameters
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
encrypted_values = encrypt_elementwise(pandas_array, self.client, **get_encrypt_config())
|
encrypted_values = encrypt_elementwise(pandas_array, self.client, **get_encrypt_config())
|
||||||
|
|
||||||
# Encrypt a 0 in order to represent NaN values
|
# Encrypt a 0 in order to represent NaN values
|
||||||
# Remove this once NaN values are not represented by 0 anymore
|
# Remove this once NaN values are not represented by 0 anymore
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
encrypted_nan = encrypt_value(0, self.client, **get_encrypt_config())
|
encrypted_nan = encrypt_value(0, self.client, **get_encrypt_config())
|
||||||
|
|
||||||
return EncryptedDataFrame(
|
return EncryptedDataFrame(
|
||||||
|
|||||||
@@ -245,7 +245,7 @@ class EncryptedDataFrame:
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Once multi-operator is supported, make sure to provide relevant keys and objects
|
# Once multi-operator is supported, make sure to provide relevant keys and objects
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
joined_df = EncryptedDataFrame(
|
joined_df = EncryptedDataFrame(
|
||||||
joined_array,
|
joined_array,
|
||||||
self._encrypted_nan,
|
self._encrypted_nan,
|
||||||
@@ -271,7 +271,7 @@ class EncryptedDataFrame:
|
|||||||
evaluation_keys = serialize_evaluation_keys(self._evaluation_keys)
|
evaluation_keys = serialize_evaluation_keys(self._evaluation_keys)
|
||||||
|
|
||||||
# Avoid sending column names and string mappings to server, instead use hashes
|
# Avoid sending column names and string mappings to server, instead use hashes
|
||||||
# FIXME : https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME : https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
# Additionally, Numpy arrays are not serializable using JSON so we need to convert them
|
# Additionally, Numpy arrays are not serializable using JSON so we need to convert them
|
||||||
# to lists
|
# to lists
|
||||||
output_dict = {
|
output_dict = {
|
||||||
|
|||||||
@@ -197,7 +197,7 @@ def get_sklearn_linear_models_and_datasets(
|
|||||||
if is_model_class_in_a_list(SGDClassifier, linear_classes):
|
if is_model_class_in_a_list(SGDClassifier, linear_classes):
|
||||||
linear_classes += [
|
linear_classes += [
|
||||||
partial(SGDClassifier, fit_encrypted=False),
|
partial(SGDClassifier, fit_encrypted=False),
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4460
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4460
|
||||||
# partial(SGDClassifier, fit_encrypted=True, parameters_range=(-1, 1)),
|
# partial(SGDClassifier, fit_encrypted=True, parameters_range=(-1, 1)),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
@@ -1059,7 +1059,7 @@ class QuantizedMixingOp(QuantizedOp, is_utility=True):
|
|||||||
|
|
||||||
if lsbs_value > 0:
|
if lsbs_value > 0:
|
||||||
# Rounding to low bit-width with approximate can cause issues with overflow protection
|
# Rounding to low bit-width with approximate can cause issues with overflow protection
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4345
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4345
|
||||||
x = fhe.round_bit_pattern(
|
x = fhe.round_bit_pattern(
|
||||||
x, lsbs_to_remove=lsbs_value, exactness=exactness, overflow_protection=False
|
x, lsbs_to_remove=lsbs_value, exactness=exactness, overflow_protection=False
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -297,7 +297,7 @@ class GLWELinearLayerExecutor:
|
|||||||
_, quantized_layer = next(iter(q_module.quant_layers_dict.items()))
|
_, quantized_layer = next(iter(q_module.quant_layers_dict.items()))
|
||||||
device = x.device
|
device = x.device
|
||||||
|
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4711
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4711
|
||||||
# Static per-channel weight quantization.
|
# Static per-channel weight quantization.
|
||||||
weight_q, weight_scale, weight_zp, sum_w = self._per_channel_weight_quantization(
|
weight_q, weight_scale, weight_zp, sum_w = self._per_channel_weight_quantization(
|
||||||
weight, q_module, device
|
weight, q_module, device
|
||||||
@@ -429,7 +429,7 @@ class GLWELinearLayerExecutor:
|
|||||||
|
|
||||||
device = x.device
|
device = x.device
|
||||||
|
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4711
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4711
|
||||||
# Dynamic quantization for weights and input.
|
# Dynamic quantization for weights and input.
|
||||||
weight_q, weight_scale, weight_zp, sum_w = self._per_channel_weight_quantization(
|
weight_q, weight_scale, weight_zp, sum_w = self._per_channel_weight_quantization(
|
||||||
weight, q_module, device
|
weight, q_module, device
|
||||||
|
|||||||
@@ -89,7 +89,7 @@ def _inspect_tree_n_bits(n_bits: Union[int, Dict[str, int]]) -> None:
|
|||||||
|
|
||||||
|
|
||||||
# Find a better naming to describe leaf quantization in tree-based models
|
# Find a better naming to describe leaf quantization in tree-based models
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4258
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4258
|
||||||
def _get_n_bits_dict_trees(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]:
|
def _get_n_bits_dict_trees(n_bits: Union[int, Dict[str, int]]) -> Dict[str, int]:
|
||||||
"""Convert the n_bits parameter into a proper dictionary for tree based-models.
|
"""Convert the n_bits parameter into a proper dictionary for tree based-models.
|
||||||
|
|
||||||
@@ -975,7 +975,7 @@ class PostTrainingAffineQuantization(ONNXConverter):
|
|||||||
if not numpy.issubdtype(values.dtype, numpy.bool_):
|
if not numpy.issubdtype(values.dtype, numpy.bool_):
|
||||||
is_signed = is_symmetric = self._check_distribution_is_symmetric_around_zero(values)
|
is_signed = is_symmetric = self._check_distribution_is_symmetric_around_zero(values)
|
||||||
# Boolean parameters are quantized to 1 bit
|
# Boolean parameters are quantized to 1 bit
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4593
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4593
|
||||||
# We should not quantize boolean parameters in the future
|
# We should not quantize boolean parameters in the future
|
||||||
else:
|
else:
|
||||||
is_signed = is_symmetric = False
|
is_signed = is_symmetric = False
|
||||||
|
|||||||
@@ -131,7 +131,7 @@ class QuantizedModule:
|
|||||||
|
|
||||||
# Set base attributes for API consistency. This could be avoided if an abstract base class
|
# Set base attributes for API consistency. This could be avoided if an abstract base class
|
||||||
# is created for both Concrete ML models and QuantizedModule
|
# is created for both Concrete ML models and QuantizedModule
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2899
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2899
|
||||||
self.input_quantizers: List[UniformQuantizer] = []
|
self.input_quantizers: List[UniformQuantizer] = []
|
||||||
self.output_quantizers: List[UniformQuantizer] = []
|
self.output_quantizers: List[UniformQuantizer] = []
|
||||||
self.fhe_circuit: Optional[Circuit] = None
|
self.fhe_circuit: Optional[Circuit] = None
|
||||||
@@ -157,7 +157,7 @@ class QuantizedModule:
|
|||||||
if self._preprocessing_module is not None:
|
if self._preprocessing_module is not None:
|
||||||
assert_true(self._preprocessing_module.onnx_preprocessing is None)
|
assert_true(self._preprocessing_module.onnx_preprocessing is None)
|
||||||
|
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4127
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4127
|
||||||
def set_reduce_sum_copy(self):
|
def set_reduce_sum_copy(self):
|
||||||
"""Set reduce sum to copy or not the inputs.
|
"""Set reduce sum to copy or not the inputs.
|
||||||
|
|
||||||
@@ -330,7 +330,7 @@ class QuantizedModule:
|
|||||||
return output_quantizers
|
return output_quantizers
|
||||||
|
|
||||||
# Remove this once we handle the re-quantization step in post-training only
|
# Remove this once we handle the re-quantization step in post-training only
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4472
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4472
|
||||||
def _add_requant_for_composition(self, composition_mapping: Optional[Dict]):
|
def _add_requant_for_composition(self, composition_mapping: Optional[Dict]):
|
||||||
"""Trigger a re-quantization step for outputs using an input-output mapping for quantizers.
|
"""Trigger a re-quantization step for outputs using an input-output mapping for quantizers.
|
||||||
|
|
||||||
@@ -589,7 +589,7 @@ class QuantizedModule:
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Remove this once we handle the re-quantization step in post-training only
|
# Remove this once we handle the re-quantization step in post-training only
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4472
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4472
|
||||||
if self._composition_mapping is not None:
|
if self._composition_mapping is not None:
|
||||||
mismatch_shapes = list(
|
mismatch_shapes = list(
|
||||||
f"Output {output_i}: {q_results[output_i].shape} "
|
f"Output {output_i}: {q_results[output_i].shape} "
|
||||||
@@ -672,7 +672,7 @@ class QuantizedModule:
|
|||||||
# If the virtual library method should be used
|
# If the virtual library method should be used
|
||||||
# For now, use the virtual library when simulating
|
# For now, use the virtual library when simulating
|
||||||
# circuits that use CRT encoding because the official simulation is too slow
|
# circuits that use CRT encoding because the official simulation is too slow
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4391
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4391
|
||||||
if USE_OLD_VL or is_crt_encoding:
|
if USE_OLD_VL or is_crt_encoding:
|
||||||
predict_method = partial(
|
predict_method = partial(
|
||||||
self.fhe_circuit.graph, p_error=self.fhe_circuit.p_error
|
self.fhe_circuit.graph, p_error=self.fhe_circuit.p_error
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
# pylint: disable=too-many-lines
|
# pylint: disable=too-many-lines
|
||||||
|
|
||||||
# This file is too long and should be split
|
# This file is too long and should be split
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/1018
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/1018
|
||||||
|
|
||||||
from typing import Any, Dict, Optional, Sequence, Set, Union
|
from typing import Any, Dict, Optional, Sequence, Set, Union
|
||||||
|
|
||||||
@@ -180,7 +180,7 @@ class QuantizedGemm(QuantizedMixingOp):
|
|||||||
# If self.constant_inputs is empty this is an encrypted gemm
|
# If self.constant_inputs is empty this is an encrypted gemm
|
||||||
# There might be caveats here
|
# There might be caveats here
|
||||||
# (for example when one of the input is passed in clear with encrypted statuses.)
|
# (for example when one of the input is passed in clear with encrypted statuses.)
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4132
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4132
|
||||||
is_encrypted_gemm = isinstance(self.constant_inputs, dict) and not self.constant_inputs
|
is_encrypted_gemm = isinstance(self.constant_inputs, dict) and not self.constant_inputs
|
||||||
|
|
||||||
# If alpha != 1 or beta not in [0, 1], this function must be modified
|
# If alpha != 1 or beta not in [0, 1], this function must be modified
|
||||||
@@ -241,7 +241,7 @@ class QuantizedGemm(QuantizedMixingOp):
|
|||||||
p = input2_q_values.shape[-2]
|
p = input2_q_values.shape[-2]
|
||||||
|
|
||||||
# Remove the manual matrix multiplication when we can handle input precision with rounding
|
# Remove the manual matrix multiplication when we can handle input precision with rounding
|
||||||
# FIXME: https://github.com/zama-ai/concrete-internal/issues/512
|
# FIXME: https://github.com/luxfi/concrete-internal/issues/512
|
||||||
def enc_mul(x, y):
|
def enc_mul(x, y):
|
||||||
r"""Encrypted multiplication of two input arrays.
|
r"""Encrypted multiplication of two input arrays.
|
||||||
|
|
||||||
@@ -284,7 +284,7 @@ class QuantizedGemm(QuantizedMixingOp):
|
|||||||
return add_pow_divide - sub_pow_divide
|
return add_pow_divide - sub_pow_divide
|
||||||
|
|
||||||
# Remove the manual matrix multiplication when we can handle input precision with rounding
|
# Remove the manual matrix multiplication when we can handle input precision with rounding
|
||||||
# FIXME: https://github.com/zama-ai/concrete-internal/issues/512
|
# FIXME: https://github.com/luxfi/concrete-internal/issues/512
|
||||||
def matmul(a, b):
|
def matmul(a, b):
|
||||||
"""Matrix multiplication of two input arrays, supporting 2D or 3D.
|
"""Matrix multiplication of two input arrays, supporting 2D or 3D.
|
||||||
|
|
||||||
@@ -350,7 +350,7 @@ class QuantizedGemm(QuantizedMixingOp):
|
|||||||
return c
|
return c
|
||||||
|
|
||||||
# Remove the manual matrix multiplication when we can handle input precision with rounding
|
# Remove the manual matrix multiplication when we can handle input precision with rounding
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4127
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4127
|
||||||
@univariate
|
@univariate
|
||||||
def copy_function(x):
|
def copy_function(x):
|
||||||
return x
|
return x
|
||||||
@@ -933,7 +933,7 @@ class QuantizedConv(QuantizedMixingOp):
|
|||||||
dilations = self.dilations
|
dilations = self.dilations
|
||||||
|
|
||||||
# Workaround for handling torch's Conv1d operator until it is supported by Concrete Python
|
# Workaround for handling torch's Conv1d operator until it is supported by Concrete Python
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4117
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4117
|
||||||
if is_conv1d:
|
if is_conv1d:
|
||||||
q_input_pad = numpy.expand_dims(q_input_pad, axis=-2)
|
q_input_pad = numpy.expand_dims(q_input_pad, axis=-2)
|
||||||
q_weights_values = numpy.expand_dims(q_weights_values, axis=-2)
|
q_weights_values = numpy.expand_dims(q_weights_values, axis=-2)
|
||||||
@@ -996,7 +996,7 @@ class QuantizedConv(QuantizedMixingOp):
|
|||||||
numpy_q_out = conv_wx
|
numpy_q_out = conv_wx
|
||||||
|
|
||||||
# Workaround for handling torch's Conv1d operator until it is supported by Concrete Python
|
# Workaround for handling torch's Conv1d operator until it is supported by Concrete Python
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4117
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4117
|
||||||
if is_conv1d:
|
if is_conv1d:
|
||||||
numpy_q_out = numpy.squeeze(numpy_q_out, axis=-2)
|
numpy_q_out = numpy.squeeze(numpy_q_out, axis=-2)
|
||||||
|
|
||||||
@@ -1680,7 +1680,7 @@ class QuantizedDiv(QuantizedMixingOp):
|
|||||||
), "Div calibrate does not support analytical calibration for now"
|
), "Div calibrate does not support analytical calibration for now"
|
||||||
assert isinstance(inputs[1], numpy.ndarray)
|
assert isinstance(inputs[1], numpy.ndarray)
|
||||||
|
|
||||||
# FIXME https://github.com/zama-ai/concrete-ml-internal/issues/4556
|
# FIXME https://github.com/luxfi/concrete-ml-internal/issues/4556
|
||||||
min_non_zero_index = numpy.abs(inputs[1]).argmin(axis=None)
|
min_non_zero_index = numpy.abs(inputs[1]).argmin(axis=None)
|
||||||
min_non_zero_value = inputs[1].flat[min_non_zero_index]
|
min_non_zero_value = inputs[1].flat[min_non_zero_index]
|
||||||
|
|
||||||
@@ -1738,7 +1738,7 @@ class QuantizedDiv(QuantizedMixingOp):
|
|||||||
# Re-quantize the inverse using the same quantization parameters as q_input_1
|
# Re-quantize the inverse using the same quantization parameters as q_input_1
|
||||||
# mypy
|
# mypy
|
||||||
assert self.divider_quantizer is not None
|
assert self.divider_quantizer is not None
|
||||||
# FIXME https://github.com/zama-ai/concrete-ml-internal/issues/4556
|
# FIXME https://github.com/luxfi/concrete-ml-internal/issues/4556
|
||||||
q_input_1_inv_rescaled = self.divider_quantizer.quant(input_1_inv)
|
q_input_1_inv_rescaled = self.divider_quantizer.quant(input_1_inv)
|
||||||
|
|
||||||
# The product of quantized encrypted integer values
|
# The product of quantized encrypted integer values
|
||||||
@@ -1775,7 +1775,7 @@ class QuantizedDiv(QuantizedMixingOp):
|
|||||||
# Apply Concrete rounding (if relevant)
|
# Apply Concrete rounding (if relevant)
|
||||||
product_q_values = self.cnp_round(product_q_values, calibrate_rounding)
|
product_q_values = self.cnp_round(product_q_values, calibrate_rounding)
|
||||||
|
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4546
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4546
|
||||||
# De-quantize the product
|
# De-quantize the product
|
||||||
dequant_product = (product_q_values - new_zero_point) * new_scale
|
dequant_product = (product_q_values - new_zero_point) * new_scale
|
||||||
|
|
||||||
@@ -1848,7 +1848,7 @@ class QuantizedMul(QuantizedMixingOp):
|
|||||||
|
|
||||||
# Remove the manual encrypted multiplication when we
|
# Remove the manual encrypted multiplication when we
|
||||||
# can handle input precision with rounding
|
# can handle input precision with rounding
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4127
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4127
|
||||||
@univariate
|
@univariate
|
||||||
def copy_function(x):
|
def copy_function(x):
|
||||||
return x
|
return x
|
||||||
@@ -2167,7 +2167,7 @@ class QuantizedReduceSum(QuantizedMixingOp):
|
|||||||
with tag(self.op_instance_name):
|
with tag(self.op_instance_name):
|
||||||
|
|
||||||
# Need to copy to prevent the following sum to raise precision of the input
|
# Need to copy to prevent the following sum to raise precision of the input
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4127
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4127
|
||||||
@univariate
|
@univariate
|
||||||
def copy_function(x):
|
def copy_function(x):
|
||||||
return x
|
return x
|
||||||
@@ -2232,7 +2232,7 @@ class QuantizedBrevitasQuant(QuantizedOp):
|
|||||||
_impl_for_op_named: str = "onnx.brevitas.Quant"
|
_impl_for_op_named: str = "onnx.brevitas.Quant"
|
||||||
# Note that this should be reset when the correctness test that finds
|
# Note that this should be reset when the correctness test that finds
|
||||||
# all mismatches between Concrete ML and Brevitas is fixed
|
# all mismatches between Concrete ML and Brevitas is fixed
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2373
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2373
|
||||||
quantize_inputs_with_model_outputs_precision = True
|
quantize_inputs_with_model_outputs_precision = True
|
||||||
output_quant_opts: QuantizationOptions
|
output_quant_opts: QuantizationOptions
|
||||||
|
|
||||||
@@ -2300,7 +2300,7 @@ class QuantizedBrevitasQuant(QuantizedOp):
|
|||||||
self.is_signed = bool(attrs["signed"])
|
self.is_signed = bool(attrs["signed"])
|
||||||
self.is_narrow = bool(attrs["narrow"])
|
self.is_narrow = bool(attrs["narrow"])
|
||||||
|
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4544
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4544
|
||||||
# Remove this workaround when brevitas export is fixed
|
# Remove this workaround when brevitas export is fixed
|
||||||
if self.is_signed is False and self.is_narrow is True:
|
if self.is_signed is False and self.is_narrow is True:
|
||||||
self.is_signed = True
|
self.is_signed = True
|
||||||
|
|||||||
@@ -526,7 +526,7 @@ class UniformQuantizationParameters:
|
|||||||
|
|
||||||
|
|
||||||
# Change UniformQuantizer inheritance from UniformQuantizationParameters to composition.
|
# Change UniformQuantizer inheritance from UniformQuantizationParameters to composition.
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/1434
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/1434
|
||||||
class UniformQuantizer(UniformQuantizationParameters, QuantizationOptions, MinMaxQuantizationStats):
|
class UniformQuantizer(UniformQuantizationParameters, QuantizationOptions, MinMaxQuantizationStats):
|
||||||
"""Uniform quantizer.
|
"""Uniform quantizer.
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ Only PyTorch neural networks and Concrete built-in models are supported.
|
|||||||
- Quantized aware trained model are supported using Brevitas framework
|
- Quantized aware trained model are supported using Brevitas framework
|
||||||
- Torch models can be converted into post-trained quantized models
|
- Torch models can be converted into post-trained quantized models
|
||||||
|
|
||||||
The `p_error` represents an essential hyper-parameter in the FHE computation at Zama.
|
The `p_error` represents an essential hyper-parameter in the FHE computation at Lux.
|
||||||
As it impacts the speed of the FHE computations and the model's performance.
|
As it impacts the speed of the FHE computations and the model's performance.
|
||||||
|
|
||||||
In this script, we provide an approach to find out an optimal `p_error`, which would offer
|
In this script, we provide an approach to find out an optimal `p_error`, which would offer
|
||||||
@@ -16,7 +16,7 @@ scheme allows to perform 2 types of operations
|
|||||||
- Linear operations: additions and multiplications
|
- Linear operations: additions and multiplications
|
||||||
- Non-linear operation: uni-variate activation functions
|
- Non-linear operation: uni-variate activation functions
|
||||||
|
|
||||||
At Zama, non-linear operations are represented by table lookup (TLU), which are implemented
|
At Lux, non-linear operations are represented by table lookup (TLU), which are implemented
|
||||||
through the Programmable Bootstrapping technology (PBS).
|
through the Programmable Bootstrapping technology (PBS).
|
||||||
A single PBS operation has `p_error` chances of being incorrect.
|
A single PBS operation has `p_error` chances of being incorrect.
|
||||||
|
|
||||||
|
|||||||
@@ -91,7 +91,7 @@ class LogisticRegressionTraining(torch.nn.Module):
|
|||||||
bias = bias * torch.zeros(bias.shape)
|
bias = bias * torch.zeros(bias.shape)
|
||||||
|
|
||||||
# Should we clip the parameters to the min-max values?
|
# Should we clip the parameters to the min-max values?
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4206
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4206
|
||||||
|
|
||||||
# (1, n_features, n_targets), (1, n_targets, 1)
|
# (1, n_features, n_targets), (1, n_targets, 1)
|
||||||
return weights, bias
|
return weights, bias
|
||||||
|
|||||||
@@ -203,7 +203,7 @@ class BaseEstimator:
|
|||||||
# scikit-learn model (once fitted), retrieve its value
|
# scikit-learn model (once fitted), retrieve its value
|
||||||
# Enable non-training attributes as well once Concrete ML models initialize their
|
# Enable non-training attributes as well once Concrete ML models initialize their
|
||||||
# underlying scikit-learn models during initialization
|
# underlying scikit-learn models during initialization
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||||
if (
|
if (
|
||||||
attr.endswith("_")
|
attr.endswith("_")
|
||||||
and not attr.endswith("__")
|
and not attr.endswith("__")
|
||||||
@@ -221,7 +221,7 @@ class BaseEstimator:
|
|||||||
# We need to specifically call the default __setattr__ method as QNN models still inherit from
|
# We need to specifically call the default __setattr__ method as QNN models still inherit from
|
||||||
# skorch, which provides its own __setattr__ implementation and creates a cyclic loop
|
# skorch, which provides its own __setattr__ implementation and creates a cyclic loop
|
||||||
# with __getattr__. Removing this inheritance once and for all should fix the issue
|
# with __getattr__. Removing this inheritance once and for all should fix the issue
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||||
def __setattr__(self, name: str, value: Any):
|
def __setattr__(self, name: str, value: Any):
|
||||||
"""Set the value as a model attribute.
|
"""Set the value as a model attribute.
|
||||||
|
|
||||||
@@ -285,7 +285,7 @@ class BaseEstimator:
|
|||||||
|
|
||||||
The FHE circuit combines computational graph, mlir, client and server into a single object.
|
The FHE circuit combines computational graph, mlir, client and server into a single object.
|
||||||
More information available in Concrete documentation
|
More information available in Concrete documentation
|
||||||
(https://docs.zama.ai/concrete/get-started/terminology)
|
(https://docs.lux.network/concrete/get-started/terminology)
|
||||||
Is None if the model is not fitted.
|
Is None if the model is not fitted.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
@@ -365,7 +365,7 @@ class BaseEstimator:
|
|||||||
# Here, the `get_params` method is the `BaseEstimator.get_params` method from scikit-learn,
|
# Here, the `get_params` method is the `BaseEstimator.get_params` method from scikit-learn,
|
||||||
# which will become available once a subclass inherits from it. We therefore disable both
|
# which will become available once a subclass inherits from it. We therefore disable both
|
||||||
# pylint and mypy as this behavior is expected
|
# pylint and mypy as this behavior is expected
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||||
# pylint: disable-next=no-member
|
# pylint: disable-next=no-member
|
||||||
params = super().get_params(deep=deep) # type: ignore[misc]
|
params = super().get_params(deep=deep) # type: ignore[misc]
|
||||||
|
|
||||||
@@ -394,7 +394,7 @@ class BaseEstimator:
|
|||||||
# Initialize the underlying scikit-learn model if it has not already been done or if
|
# Initialize the underlying scikit-learn model if it has not already been done or if
|
||||||
# `warm_start` is set to False (for neural networks)
|
# `warm_start` is set to False (for neural networks)
|
||||||
# This model should be directly initialized in the model's __init__ method instead
|
# This model should be directly initialized in the model's __init__ method instead
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||||
if self.sklearn_model is None or not getattr(self, "warm_start", False):
|
if self.sklearn_model is None or not getattr(self, "warm_start", False):
|
||||||
# Retrieve the init parameters
|
# Retrieve the init parameters
|
||||||
params = self.get_sklearn_params()
|
params = self.get_sklearn_params()
|
||||||
@@ -828,7 +828,7 @@ class BaseEstimator:
|
|||||||
# If the virtual library method should be used
|
# If the virtual library method should be used
|
||||||
# For now, use the virtual library when simulating
|
# For now, use the virtual library when simulating
|
||||||
# circuits that use CRT encoding because the official simulation is too slow
|
# circuits that use CRT encoding because the official simulation is too slow
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4391
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4391
|
||||||
if USE_OLD_VL or is_crt_encoding:
|
if USE_OLD_VL or is_crt_encoding:
|
||||||
predict_method = partial(
|
predict_method = partial(
|
||||||
self.fhe_circuit.graph, p_error=self.fhe_circuit.p_error
|
self.fhe_circuit.graph, p_error=self.fhe_circuit.p_error
|
||||||
@@ -904,7 +904,7 @@ class BaseClassifier(BaseEstimator):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
# Remove in our next release major release
|
# Remove in our next release major release
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3994
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3994
|
||||||
@property
|
@property
|
||||||
def target_classes_(self) -> Optional[numpy.ndarray]: # pragma: no cover
|
def target_classes_(self) -> Optional[numpy.ndarray]: # pragma: no cover
|
||||||
"""Get the model's classes.
|
"""Get the model's classes.
|
||||||
@@ -924,7 +924,7 @@ class BaseClassifier(BaseEstimator):
|
|||||||
return self.classes_
|
return self.classes_
|
||||||
|
|
||||||
# Remove in our next release major release
|
# Remove in our next release major release
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3994
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3994
|
||||||
@property
|
@property
|
||||||
def n_classes_(self) -> int: # pragma: no cover
|
def n_classes_(self) -> int: # pragma: no cover
|
||||||
"""Get the model's number of classes.
|
"""Get the model's number of classes.
|
||||||
@@ -958,7 +958,7 @@ class BaseClassifier(BaseEstimator):
|
|||||||
assert_true(len(classes) > 1, "You must provide at least 2 classes in y.")
|
assert_true(len(classes) > 1, "You must provide at least 2 classes in y.")
|
||||||
|
|
||||||
# Change to composition in order to avoid diamond inheritance and indirect super() calls
|
# Change to composition in order to avoid diamond inheritance and indirect super() calls
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3249
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3249
|
||||||
return super().fit(X, y, **fit_parameters) # type: ignore[safe-super]
|
return super().fit(X, y, **fit_parameters) # type: ignore[safe-super]
|
||||||
|
|
||||||
def predict_proba(self, X: Data, fhe: Union[FheMode, str] = FheMode.DISABLE) -> numpy.ndarray:
|
def predict_proba(self, X: Data, fhe: Union[FheMode, str] = FheMode.DISABLE) -> numpy.ndarray:
|
||||||
@@ -1015,7 +1015,7 @@ class BaseClassifier(BaseEstimator):
|
|||||||
# Pylint complains that this method does not override the `dump_dict` and `load_dict` methods. This
|
# Pylint complains that this method does not override the `dump_dict` and `load_dict` methods. This
|
||||||
# is expected as the QuantizedTorchEstimatorMixin class is not supposed to be used as such. This
|
# is expected as the QuantizedTorchEstimatorMixin class is not supposed to be used as such. This
|
||||||
# disable could probably be removed when refactoring the serialization of models
|
# disable could probably be removed when refactoring the serialization of models
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3250
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3250
|
||||||
# pylint: disable-next=abstract-method,too-many-instance-attributes
|
# pylint: disable-next=abstract-method,too-many-instance-attributes
|
||||||
class QuantizedTorchEstimatorMixin(BaseEstimator):
|
class QuantizedTorchEstimatorMixin(BaseEstimator):
|
||||||
"""Mixin that provides quantization for a torch module and follows the Estimator API."""
|
"""Mixin that provides quantization for a torch module and follows the Estimator API."""
|
||||||
@@ -1099,7 +1099,7 @@ class QuantizedTorchEstimatorMixin(BaseEstimator):
|
|||||||
# Here, the `get_params` method is the `NeuralNet.get_params` method from skorch, which
|
# Here, the `get_params` method is the `NeuralNet.get_params` method from skorch, which
|
||||||
# will become available once a subclass inherits from it. We therefore disable both pylint
|
# will become available once a subclass inherits from it. We therefore disable both pylint
|
||||||
# and mypy as this behavior is expected
|
# and mypy as this behavior is expected
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||||
# pylint: disable-next=no-member
|
# pylint: disable-next=no-member
|
||||||
params = super().get_params(deep) # type: ignore[misc]
|
params = super().get_params(deep) # type: ignore[misc]
|
||||||
|
|
||||||
@@ -1118,7 +1118,7 @@ class QuantizedTorchEstimatorMixin(BaseEstimator):
|
|||||||
# Here, the `get_params` method is the `NeuralNet.get_params` method from skorch, which
|
# Here, the `get_params` method is the `NeuralNet.get_params` method from skorch, which
|
||||||
# will become available once a subclass inherits from it. We therefore disable both pylint
|
# will become available once a subclass inherits from it. We therefore disable both pylint
|
||||||
# and mypy as this behavior is expected
|
# and mypy as this behavior is expected
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||||
# pylint: disable-next=no-member
|
# pylint: disable-next=no-member
|
||||||
params = super().get_params(deep=deep) # type: ignore[misc]
|
params = super().get_params(deep=deep) # type: ignore[misc]
|
||||||
|
|
||||||
@@ -1478,7 +1478,7 @@ class QuantizedTorchEstimatorMixin(BaseEstimator):
|
|||||||
# The .module_ was initialized manually, prevent .fit (for both skorch and Concrete ML)
|
# The .module_ was initialized manually, prevent .fit (for both skorch and Concrete ML)
|
||||||
# from creating a new one
|
# from creating a new one
|
||||||
# Setting both attributes could be avoided by initializing `sklearn_model` in __init__
|
# Setting both attributes could be avoided by initializing `sklearn_model` in __init__
|
||||||
# # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
# # FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||||
pruned_model.warm_start = True
|
pruned_model.warm_start = True
|
||||||
pruned_model.sklearn_model.warm_start = True
|
pruned_model.sklearn_model.warm_start = True
|
||||||
|
|
||||||
@@ -1572,7 +1572,7 @@ class BaseTreeEstimatorMixin(BaseEstimator, sklearn.base.BaseEstimator, ABC):
|
|||||||
|
|
||||||
# Get the onnx model, all operations needed to load it properly will be done on it.
|
# Get the onnx model, all operations needed to load it properly will be done on it.
|
||||||
n_features = model.n_features_in_
|
n_features = model.n_features_in_
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4545
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4545
|
||||||
# Execute with 2 example for efficiency in large data scenarios to prevent slowdown
|
# Execute with 2 example for efficiency in large data scenarios to prevent slowdown
|
||||||
# but also to work around the HB export issue.
|
# but also to work around the HB export issue.
|
||||||
dummy_input = numpy.zeros((2, n_features))
|
dummy_input = numpy.zeros((2, n_features))
|
||||||
@@ -1891,7 +1891,7 @@ class SklearnLinearModelMixin(BaseEstimator, sklearn.base.BaseEstimator, ABC):
|
|||||||
# scikit-learn models
|
# scikit-learn models
|
||||||
# This should be fixed once Concrete ML models initialize their underlying scikit-learn
|
# This should be fixed once Concrete ML models initialize their underlying scikit-learn
|
||||||
# models during initialization
|
# models during initialization
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||||
model = cls(n_bits=n_bits, **init_params)
|
model = cls(n_bits=n_bits, **init_params)
|
||||||
|
|
||||||
# Update the underlying scikit-learn model with the given fitted one
|
# Update the underlying scikit-learn model with the given fitted one
|
||||||
@@ -2154,7 +2154,7 @@ class SklearnSGDRegressorMixin(SklearnLinearRegressorMixin):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
# Remove once Hummingbird supports SGDRegressor
|
# Remove once Hummingbird supports SGDRegressor
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4100
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4100
|
||||||
def _set_onnx_model(self, test_input: numpy.ndarray) -> None:
|
def _set_onnx_model(self, test_input: numpy.ndarray) -> None:
|
||||||
"""Retrieve the model's ONNX graph using Hummingbird conversion.
|
"""Retrieve the model's ONNX graph using Hummingbird conversion.
|
||||||
|
|
||||||
@@ -2186,7 +2186,7 @@ class SklearnSGDClassifierMixin(SklearnLinearClassifierMixin):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
# Remove once Hummingbird supports SGDClassifier
|
# Remove once Hummingbird supports SGDClassifier
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4100
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4100
|
||||||
def _set_onnx_model(self, test_input: numpy.ndarray) -> None:
|
def _set_onnx_model(self, test_input: numpy.ndarray) -> None:
|
||||||
"""Retrieve the model's ONNX graph using Hummingbird conversion.
|
"""Retrieve the model's ONNX graph using Hummingbird conversion.
|
||||||
|
|
||||||
|
|||||||
@@ -139,7 +139,7 @@ class _GeneralizedLinearRegressor(SklearnLinearRegressorMixin):
|
|||||||
|
|
||||||
def get_sklearn_params(self, deep: bool = True) -> dict:
|
def get_sklearn_params(self, deep: bool = True) -> dict:
|
||||||
# Here, the `get_params` method is the `BaseEstimator.get_params` method from scikit-learn
|
# Here, the `get_params` method is the `BaseEstimator.get_params` method from scikit-learn
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||||
params = super().get_params(deep=deep) # type: ignore[misc]
|
params = super().get_params(deep=deep) # type: ignore[misc]
|
||||||
|
|
||||||
# Remove the parameters added by Concrete ML
|
# Remove the parameters added by Concrete ML
|
||||||
|
|||||||
@@ -180,7 +180,7 @@ class SGDClassifier(SklearnSGDClassifierMixin):
|
|||||||
# into account when training, so we can just modify them manually.
|
# into account when training, so we can just modify them manually.
|
||||||
# The number of bits used for training should be adjusted according to n-bits
|
# The number of bits used for training should be adjusted according to n-bits
|
||||||
# but for now we use this hardcoded values.
|
# but for now we use this hardcoded values.
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4205
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4205
|
||||||
self.n_bits_training = 6
|
self.n_bits_training = 6
|
||||||
self.rounding_training = 7
|
self.rounding_training = 7
|
||||||
self.learning_rate_value = 1.0
|
self.learning_rate_value = 1.0
|
||||||
@@ -258,7 +258,7 @@ class SGDClassifier(SklearnSGDClassifierMixin):
|
|||||||
|
|
||||||
def get_sklearn_params(self, deep: bool = True) -> dict:
|
def get_sklearn_params(self, deep: bool = True) -> dict:
|
||||||
# Here, the `get_params` method is the `BaseEstimator.get_params` method from scikit-learn
|
# Here, the `get_params` method is the `BaseEstimator.get_params` method from scikit-learn
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||||
params = super().get_params(deep=deep) # type: ignore[misc]
|
params = super().get_params(deep=deep) # type: ignore[misc]
|
||||||
|
|
||||||
# Remove the parameters added by Concrete ML
|
# Remove the parameters added by Concrete ML
|
||||||
@@ -314,7 +314,7 @@ class SGDClassifier(SklearnSGDClassifierMixin):
|
|||||||
|
|
||||||
# Generate the target values to consider for compilation
|
# Generate the target values to consider for compilation
|
||||||
# Update this once we support multi-class
|
# Update this once we support multi-class
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4182
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4182
|
||||||
y_compile_set = numpy.empty((compile_size, self.batch_size, n_targets))
|
y_compile_set = numpy.empty((compile_size, self.batch_size, n_targets))
|
||||||
|
|
||||||
# Generate the weight values to consider for compilation
|
# Generate the weight values to consider for compilation
|
||||||
@@ -501,7 +501,7 @@ class SGDClassifier(SklearnSGDClassifierMixin):
|
|||||||
# that this is the partial fit's first call)
|
# that this is the partial fit's first call)
|
||||||
if (not is_partial_fit) or (self.training_quantized_module is None):
|
if (not is_partial_fit) or (self.training_quantized_module is None):
|
||||||
# Update this once we support multi-class
|
# Update this once we support multi-class
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4182
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4182
|
||||||
# We need to define this here and not in the init otherwise this breaks
|
# We need to define this here and not in the init otherwise this breaks
|
||||||
# because scikit-learn assumes that as soon as the attribute exists
|
# because scikit-learn assumes that as soon as the attribute exists
|
||||||
# the model is fitted
|
# the model is fitted
|
||||||
@@ -741,7 +741,7 @@ class SGDClassifier(SklearnSGDClassifierMixin):
|
|||||||
|
|
||||||
# Initialize the underlying scikit-learn model if it has not already been done
|
# Initialize the underlying scikit-learn model if it has not already been done
|
||||||
# This model should be directly initialized in the model's __init__ method instead
|
# This model should be directly initialized in the model's __init__ method instead
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||||
if self.sklearn_model is None:
|
if self.sklearn_model is None:
|
||||||
|
|
||||||
# Retrieve the init parameters
|
# Retrieve the init parameters
|
||||||
@@ -913,7 +913,7 @@ class SGDClassifier(SklearnSGDClassifierMixin):
|
|||||||
|
|
||||||
else:
|
else:
|
||||||
# Expose and implement partial_fit for clear training
|
# Expose and implement partial_fit for clear training
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4184
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4184
|
||||||
raise NotImplementedError("Partial fit is not currently supported for clear training.")
|
raise NotImplementedError("Partial fit is not currently supported for clear training.")
|
||||||
|
|
||||||
def post_processing(self, y_preds: numpy.ndarray) -> numpy.ndarray:
|
def post_processing(self, y_preds: numpy.ndarray) -> numpy.ndarray:
|
||||||
|
|||||||
@@ -121,7 +121,7 @@ class KNeighborsClassifier(SklearnKNeighborsClassifierMixin):
|
|||||||
return obj
|
return obj
|
||||||
|
|
||||||
# KNeighborsClassifier does not provide a predict_proba method for now
|
# KNeighborsClassifier does not provide a predict_proba method for now
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||||
def predict_proba(self, X: Data, fhe: Union[FheMode, str] = FheMode.DISABLE) -> numpy.ndarray:
|
def predict_proba(self, X: Data, fhe: Union[FheMode, str] = FheMode.DISABLE) -> numpy.ndarray:
|
||||||
"""Predict class probabilities.
|
"""Predict class probabilities.
|
||||||
|
|
||||||
@@ -144,7 +144,7 @@ class KNeighborsClassifier(SklearnKNeighborsClassifierMixin):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# KNeighborsClassifier does not provide a kneighbors method
|
# KNeighborsClassifier does not provide a kneighbors method
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4080
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4080
|
||||||
def kneighbors(self, X: Data) -> numpy.ndarray:
|
def kneighbors(self, X: Data) -> numpy.ndarray:
|
||||||
"""Return the knearest distances and their respective indices for each query point.
|
"""Return the knearest distances and their respective indices for each query point.
|
||||||
|
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ ATTRIBUTE_PREFIXES = [
|
|||||||
|
|
||||||
|
|
||||||
# We should also check that the `module__n_layers` parameter is properly set
|
# We should also check that the `module__n_layers` parameter is properly set
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3553
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3553
|
||||||
def _check_qnn_kwargs(input_kwargs: Dict[str, Any]) -> None:
|
def _check_qnn_kwargs(input_kwargs: Dict[str, Any]) -> None:
|
||||||
"""Check that a QNN model is not constructed with automatically computed parameters.
|
"""Check that a QNN model is not constructed with automatically computed parameters.
|
||||||
|
|
||||||
@@ -182,7 +182,7 @@ class NeuralNetRegressor(QuantizedTorchEstimatorMixin, skorch.regressor.NeuralNe
|
|||||||
# method accessible by anyone, without having any FHE implementation. As this could create some
|
# method accessible by anyone, without having any FHE implementation. As this could create some
|
||||||
# confusion, a NotImplementedError is raised. This issue could be fixed by making these classes
|
# confusion, a NotImplementedError is raised. This issue could be fixed by making these classes
|
||||||
# not inherit from skorch.
|
# not inherit from skorch.
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||||
def predict_proba(self, X: Data, fhe: Union[FheMode, str] = FheMode.DISABLE) -> numpy.ndarray:
|
def predict_proba(self, X: Data, fhe: Union[FheMode, str] = FheMode.DISABLE) -> numpy.ndarray:
|
||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
"The `predict_proba` method is not implemented for neural network regressors. Please "
|
"The `predict_proba` method is not implemented for neural network regressors. Please "
|
||||||
@@ -238,7 +238,7 @@ class NeuralNetRegressor(QuantizedTorchEstimatorMixin, skorch.regressor.NeuralNe
|
|||||||
metadata["post_processing_params"] = self.post_processing_params
|
metadata["post_processing_params"] = self.post_processing_params
|
||||||
|
|
||||||
# skorch attributes that cannot be serialized
|
# skorch attributes that cannot be serialized
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3550
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3550
|
||||||
# Disable mypy as running isinstance with a Callable type unexpectedly raises an issue:
|
# Disable mypy as running isinstance with a Callable type unexpectedly raises an issue:
|
||||||
# https://github.com/python/mypy/issues/3060
|
# https://github.com/python/mypy/issues/3060
|
||||||
if isinstance(self.train_split, Callable) and not isinstance( # type: ignore[arg-type]
|
if isinstance(self.train_split, Callable) and not isinstance( # type: ignore[arg-type]
|
||||||
@@ -299,7 +299,7 @@ class NeuralNetRegressor(QuantizedTorchEstimatorMixin, skorch.regressor.NeuralNe
|
|||||||
|
|
||||||
# skorch special arguments
|
# skorch special arguments
|
||||||
# Coverage is disabled here as refactoring the serialization feature should remove this
|
# Coverage is disabled here as refactoring the serialization feature should remove this
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3250
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3250
|
||||||
for attribute_prefix in ATTRIBUTE_PREFIXES: # pragma: no cover
|
for attribute_prefix in ATTRIBUTE_PREFIXES: # pragma: no cover
|
||||||
for qnn_attribute in vars(self):
|
for qnn_attribute in vars(self):
|
||||||
if qnn_attribute.startswith(f"{attribute_prefix}__"):
|
if qnn_attribute.startswith(f"{attribute_prefix}__"):
|
||||||
@@ -352,7 +352,7 @@ class NeuralNetRegressor(QuantizedTorchEstimatorMixin, skorch.regressor.NeuralNe
|
|||||||
|
|
||||||
# skorch special arguments
|
# skorch special arguments
|
||||||
# Coverage is disabled here as refactoring the serialization feature should remove this
|
# Coverage is disabled here as refactoring the serialization feature should remove this
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3250
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3250
|
||||||
for attribute_prefix in ATTRIBUTE_PREFIXES: # pragma: no cover
|
for attribute_prefix in ATTRIBUTE_PREFIXES: # pragma: no cover
|
||||||
for qnn_attribute, qnn_value in metadata.items():
|
for qnn_attribute, qnn_value in metadata.items():
|
||||||
if qnn_attribute.startswith(f"{attribute_prefix}__"):
|
if qnn_attribute.startswith(f"{attribute_prefix}__"):
|
||||||
@@ -550,7 +550,7 @@ class NeuralNetClassifier(
|
|||||||
metadata["post_processing_params"] = self.post_processing_params
|
metadata["post_processing_params"] = self.post_processing_params
|
||||||
|
|
||||||
# skorch attributes that cannot be serialized
|
# skorch attributes that cannot be serialized
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3550
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3550
|
||||||
# Disable mypy as running isinstance with a Callable type unexpectedly raises an issue:
|
# Disable mypy as running isinstance with a Callable type unexpectedly raises an issue:
|
||||||
# https://github.com/python/mypy/issues/3060
|
# https://github.com/python/mypy/issues/3060
|
||||||
if isinstance(self.train_split, Callable) and not isinstance( # type: ignore[arg-type]
|
if isinstance(self.train_split, Callable) and not isinstance( # type: ignore[arg-type]
|
||||||
@@ -612,7 +612,7 @@ class NeuralNetClassifier(
|
|||||||
|
|
||||||
# skorch special arguments
|
# skorch special arguments
|
||||||
# Coverage is disabled here as refactoring the serialization feature should remove this
|
# Coverage is disabled here as refactoring the serialization feature should remove this
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3250
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3250
|
||||||
for attribute_prefix in ATTRIBUTE_PREFIXES: # pragma: no cover
|
for attribute_prefix in ATTRIBUTE_PREFIXES: # pragma: no cover
|
||||||
for qnn_attribute in vars(self):
|
for qnn_attribute in vars(self):
|
||||||
if qnn_attribute.startswith(f"{attribute_prefix}__"):
|
if qnn_attribute.startswith(f"{attribute_prefix}__"):
|
||||||
@@ -666,7 +666,7 @@ class NeuralNetClassifier(
|
|||||||
|
|
||||||
# skorch special arguments
|
# skorch special arguments
|
||||||
# Coverage is disabled here as refactoring the serialization feature should remove this
|
# Coverage is disabled here as refactoring the serialization feature should remove this
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3250
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3250
|
||||||
for attribute_prefix in ATTRIBUTE_PREFIXES: # pragma: no cover
|
for attribute_prefix in ATTRIBUTE_PREFIXES: # pragma: no cover
|
||||||
for qnn_attribute, qnn_value in metadata.items():
|
for qnn_attribute, qnn_value in metadata.items():
|
||||||
if qnn_attribute.startswith(f"{attribute_prefix}__"):
|
if qnn_attribute.startswith(f"{attribute_prefix}__"):
|
||||||
|
|||||||
@@ -81,7 +81,7 @@ def get_onnx_model(model, x: numpy.ndarray, framework: str) -> onnx.ModelProto:
|
|||||||
def workaround_squeeze_node_xgboost(onnx_model: onnx.ModelProto):
|
def workaround_squeeze_node_xgboost(onnx_model: onnx.ModelProto):
|
||||||
"""Workaround to fix torch issue that does not export the proper axis in the ONNX squeeze node.
|
"""Workaround to fix torch issue that does not export the proper axis in the ONNX squeeze node.
|
||||||
|
|
||||||
FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2778
|
FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2778
|
||||||
The squeeze ops does not have the proper dimensions.
|
The squeeze ops does not have the proper dimensions.
|
||||||
remove the following workaround when the issue is fixed
|
remove the following workaround when the issue is fixed
|
||||||
Add the axis attribute to the Squeeze node
|
Add the axis attribute to the Squeeze node
|
||||||
@@ -267,7 +267,7 @@ def tree_onnx_graph_preprocessing(
|
|||||||
clean_graph_at_node_op_type(onnx_model, "ReduceSum")
|
clean_graph_at_node_op_type(onnx_model, "ReduceSum")
|
||||||
|
|
||||||
if framework == "xgboost":
|
if framework == "xgboost":
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2778
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2778
|
||||||
# The squeeze ops does not have the proper dimensions.
|
# The squeeze ops does not have the proper dimensions.
|
||||||
# remove the following workaround when the issue is fixed
|
# remove the following workaround when the issue is fixed
|
||||||
# Add the axis attribute to the Squeeze node
|
# Add the axis attribute to the Squeeze node
|
||||||
@@ -369,7 +369,7 @@ def tree_to_numpy(
|
|||||||
f"framework={framework} is not supported. It must be either 'xgboost' or 'sklearn'",
|
f"framework={framework} is not supported. It must be either 'xgboost' or 'sklearn'",
|
||||||
)
|
)
|
||||||
|
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4545
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4545
|
||||||
# Execute with 2 example for efficiency in large data scenarios to prevent slowdown
|
# Execute with 2 example for efficiency in large data scenarios to prevent slowdown
|
||||||
# but also to work around the HB export issue
|
# but also to work around the HB export issue
|
||||||
onnx_model = get_onnx_model(model, x[:2] if x.shape[0] > 1 else x, framework)
|
onnx_model = get_onnx_model(model, x[:2] if x.shape[0] > 1 else x, framework)
|
||||||
@@ -403,7 +403,7 @@ def tree_to_numpy(
|
|||||||
|
|
||||||
|
|
||||||
# Remove this function once the truncate feature is released
|
# Remove this function once the truncate feature is released
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4143
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4143
|
||||||
def _compute_lsb_to_remove_for_trees(
|
def _compute_lsb_to_remove_for_trees(
|
||||||
onnx_model: onnx.ModelProto, q_x: numpy.ndarray
|
onnx_model: onnx.ModelProto, q_x: numpy.ndarray
|
||||||
) -> Tuple[int, int]:
|
) -> Tuple[int, int]:
|
||||||
|
|||||||
@@ -72,17 +72,17 @@ class XGBClassifier(BaseTreeClassifierMixin):
|
|||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
# base_score != 0.5 or None does not seem to not pass our tests
|
# base_score != 0.5 or None does not seem to not pass our tests
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/474
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/474
|
||||||
assert_true(
|
assert_true(
|
||||||
base_score in [0.5, None],
|
base_score in [0.5, None],
|
||||||
f"Currently, only 0.5 or None are supported for base_score. Got {base_score}",
|
f"Currently, only 0.5 or None are supported for base_score. Got {base_score}",
|
||||||
)
|
)
|
||||||
|
|
||||||
# See https://github.com/zama-ai/concrete-ml-internal/issues/503, there is currently
|
# See https://github.com/luxfi/concrete-ml-internal/issues/503, there is currently
|
||||||
# an issue with n_jobs != 1 on macOS
|
# an issue with n_jobs != 1 on macOS
|
||||||
#
|
#
|
||||||
# When it gets fixed, we'll remove this workaround
|
# When it gets fixed, we'll remove this workaround
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2747
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2747
|
||||||
if platform.system() == "Darwin":
|
if platform.system() == "Darwin":
|
||||||
if n_jobs != 1: # pragma: no cover
|
if n_jobs != 1: # pragma: no cover
|
||||||
warnings.warn(
|
warnings.warn(
|
||||||
@@ -218,7 +218,7 @@ class XGBClassifier(BaseTreeClassifierMixin):
|
|||||||
obj.output_quantizers = metadata["output_quantizers"]
|
obj.output_quantizers = metadata["output_quantizers"]
|
||||||
obj._fhe_ensembling = metadata["_fhe_ensembling"]
|
obj._fhe_ensembling = metadata["_fhe_ensembling"]
|
||||||
|
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4545
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4545
|
||||||
# Execute with 2 example for efficiency in large data scenarios to prevent slowdown
|
# Execute with 2 example for efficiency in large data scenarios to prevent slowdown
|
||||||
# but also to work around the HB export issue.
|
# but also to work around the HB export issue.
|
||||||
obj._tree_inference = tree_to_numpy(
|
obj._tree_inference = tree_to_numpy(
|
||||||
@@ -330,17 +330,17 @@ class XGBRegressor(BaseTreeRegressorMixin):
|
|||||||
**kwargs: Any,
|
**kwargs: Any,
|
||||||
):
|
):
|
||||||
# base_score != 0.5 or None does not seem to not pass our tests
|
# base_score != 0.5 or None does not seem to not pass our tests
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/474
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/474
|
||||||
assert_true(
|
assert_true(
|
||||||
base_score in [0.5, None],
|
base_score in [0.5, None],
|
||||||
f"Currently, only 0.5 or None are supported for base_score. Got {base_score}",
|
f"Currently, only 0.5 or None are supported for base_score. Got {base_score}",
|
||||||
)
|
)
|
||||||
|
|
||||||
# See https://github.com/zama-ai/concrete-ml-internal/issues/503, there is currently
|
# See https://github.com/luxfi/concrete-ml-internal/issues/503, there is currently
|
||||||
# an issue with n_jobs != 1 on macOS
|
# an issue with n_jobs != 1 on macOS
|
||||||
#
|
#
|
||||||
# When it gets fixed, we'll remove this workaround
|
# When it gets fixed, we'll remove this workaround
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2747
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2747
|
||||||
if platform.system() == "Darwin":
|
if platform.system() == "Darwin":
|
||||||
if n_jobs != 1: # pragma: no cover
|
if n_jobs != 1: # pragma: no cover
|
||||||
warnings.warn(
|
warnings.warn(
|
||||||
@@ -393,7 +393,7 @@ class XGBRegressor(BaseTreeRegressorMixin):
|
|||||||
def fit(self, X, y, *args, **kwargs) -> Any:
|
def fit(self, X, y, *args, **kwargs) -> Any:
|
||||||
|
|
||||||
# Hummingbird and XGBoost don't properly manage multi-outputs cases
|
# Hummingbird and XGBoost don't properly manage multi-outputs cases
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/1856
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/1856
|
||||||
|
|
||||||
assert_true(
|
assert_true(
|
||||||
(isinstance(y, list) and (not isinstance(y[0], list) or (len(y[0]) == 1)))
|
(isinstance(y, list) and (not isinstance(y[0], list) or (len(y[0]) == 1)))
|
||||||
@@ -497,7 +497,7 @@ class XGBRegressor(BaseTreeRegressorMixin):
|
|||||||
obj.output_quantizers = metadata["output_quantizers"]
|
obj.output_quantizers = metadata["output_quantizers"]
|
||||||
obj._fhe_ensembling = metadata["_fhe_ensembling"]
|
obj._fhe_ensembling = metadata["_fhe_ensembling"]
|
||||||
|
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4545
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4545
|
||||||
# Execute with 2 example for efficiency in large data scenarios to prevent slowdown
|
# Execute with 2 example for efficiency in large data scenarios to prevent slowdown
|
||||||
# but also to work around the HB export issue.
|
# but also to work around the HB export issue.
|
||||||
obj._tree_inference = tree_to_numpy(
|
obj._tree_inference = tree_to_numpy(
|
||||||
|
|||||||
@@ -141,7 +141,7 @@ def build_quantized_module(
|
|||||||
*inputset_as_numpy_tuple, keep_onnx=keep_onnx
|
*inputset_as_numpy_tuple, keep_onnx=keep_onnx
|
||||||
)
|
)
|
||||||
|
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4127
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4127
|
||||||
if reduce_sum_copy:
|
if reduce_sum_copy:
|
||||||
quantized_module.set_reduce_sum_copy()
|
quantized_module.set_reduce_sum_copy()
|
||||||
|
|
||||||
|
|||||||
@@ -57,7 +57,7 @@ def underscore_str_to_tuple(tup: str) -> Tuple:
|
|||||||
return ast.literal_eval(tup.replace("po_", "(").replace("_pc", ")").replace("_", ", "))
|
return ast.literal_eval(tup.replace("po_", "(").replace("_pc", ")").replace("_", ", "))
|
||||||
|
|
||||||
|
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3858
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3858
|
||||||
def convert_conv1d_to_linear(layer_or_module):
|
def convert_conv1d_to_linear(layer_or_module):
|
||||||
"""Convert all Conv1D layers in a module or a Conv1D layer itself to nn.Linear.
|
"""Convert all Conv1D layers in a module or a Conv1D layer itself to nn.Linear.
|
||||||
|
|
||||||
@@ -293,7 +293,7 @@ class RemoteModule(nn.Module):
|
|||||||
|
|
||||||
elif self.fhe_local_mode == HybridFHEMode.REMOTE: # pragma:no cover
|
elif self.fhe_local_mode == HybridFHEMode.REMOTE: # pragma:no cover
|
||||||
# Remote call
|
# Remote call
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4672
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4672
|
||||||
assert self.executor is None, "Remote optimized linear layers are not yet implemented"
|
assert self.executor is None, "Remote optimized linear layers are not yet implemented"
|
||||||
y = self.remote_call(x)
|
y = self.remote_call(x)
|
||||||
|
|
||||||
@@ -423,7 +423,7 @@ class HybridFHEModel:
|
|||||||
"""Replace the private modules in the model with remote layers."""
|
"""Replace the private modules in the model with remote layers."""
|
||||||
self._has_only_large_linear_layers = True
|
self._has_only_large_linear_layers = True
|
||||||
for module_name in self.module_names:
|
for module_name in self.module_names:
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3858
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3858
|
||||||
# Conv1d introduce reshaping operations which adds more TLU
|
# Conv1d introduce reshaping operations which adds more TLU
|
||||||
self.private_modules[module_name] = convert_conv1d_to_linear(
|
self.private_modules[module_name] = convert_conv1d_to_linear(
|
||||||
self.private_modules[module_name]
|
self.private_modules[module_name]
|
||||||
@@ -755,7 +755,7 @@ class HybridFHEModel:
|
|||||||
# Save the model with a specific filename
|
# Save the model with a specific filename
|
||||||
model_path = path / "model.pth"
|
model_path = path / "model.pth"
|
||||||
# Save the model state dict due to a Brevitas issue
|
# Save the model state dict due to a Brevitas issue
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4572
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4572
|
||||||
torch.save(self.model.state_dict(), model_path.resolve())
|
torch.save(self.model.state_dict(), model_path.resolve())
|
||||||
|
|
||||||
# Save the FHE circuit in the same directory
|
# Save the FHE circuit in the same directory
|
||||||
|
|||||||
@@ -439,7 +439,7 @@ class LoraTrainer:
|
|||||||
use_dynamic_quantization=use_dynamic_quantization,
|
use_dynamic_quantization=use_dynamic_quantization,
|
||||||
)
|
)
|
||||||
|
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4707
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4707
|
||||||
# Need a forward call to set the executors in remote modules
|
# Need a forward call to set the executors in remote modules
|
||||||
self.hybrid_model.set_fhe_mode("disable")
|
self.hybrid_model.set_fhe_mode("disable")
|
||||||
self.hybrid_model(inputset)
|
self.hybrid_model(inputset)
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ def test_config_sklearn(model_class, parameters, kwargs, load_data):
|
|||||||
model.compile(x, verbose=True, **kwargs)
|
model.compile(x, verbose=True, **kwargs)
|
||||||
|
|
||||||
# We still need to check that we have the expected probabilities
|
# We still need to check that we have the expected probabilities
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2206
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2206
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
@@ -102,4 +102,4 @@ def test_config_torch(model, kwargs):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# We still need to check that we have the expected probabilities
|
# We still need to check that we have the expected probabilities
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2206
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2206
|
||||||
|
|||||||
@@ -299,7 +299,7 @@ def test_serialize_valid_split(cross_validation_split, stratified, random_state)
|
|||||||
"Object of type Tensor is not JSON serializable",
|
"Object of type Tensor is not JSON serializable",
|
||||||
),
|
),
|
||||||
# Serializing a Circuit object is currently not supported
|
# Serializing a Circuit object is currently not supported
|
||||||
# FIXME: https://github.com/zama-ai/concrete-numpy-internal/issues/1841
|
# FIXME: https://github.com/luxfi/concrete-numpy-internal/issues/1841
|
||||||
pytest.param(
|
pytest.param(
|
||||||
get_a_fhe_circuit(),
|
get_a_fhe_circuit(),
|
||||||
NotImplementedError,
|
NotImplementedError,
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ from concrete.ml.torch.compile import compile_torch_model
|
|||||||
|
|
||||||
|
|
||||||
# Add encrypted training with SGDClassifier manually
|
# Add encrypted training with SGDClassifier manually
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4460
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4460
|
||||||
MODELS_AND_DATASETS = MODELS_AND_DATASETS + [
|
MODELS_AND_DATASETS = MODELS_AND_DATASETS + [
|
||||||
pytest.param(
|
pytest.param(
|
||||||
partial(SGDClassifier, fit_encrypted=True, parameters_range=(-1, 1)),
|
partial(SGDClassifier, fit_encrypted=True, parameters_range=(-1, 1)),
|
||||||
@@ -67,9 +67,9 @@ class OnDiskNetwork:
|
|||||||
|
|
||||||
|
|
||||||
# This is a known flaky test
|
# This is a known flaky test
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4014
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4014
|
||||||
# This test is disabled on CPU because it is getting stuck
|
# This test is disabled on CPU because it is getting stuck
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4737
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4737
|
||||||
# @pytest.mark.use_gpu
|
# @pytest.mark.use_gpu
|
||||||
@pytest.mark.flaky
|
@pytest.mark.flaky
|
||||||
@pytest.mark.parametrize("model_class, parameters", MODELS_AND_DATASETS)
|
@pytest.mark.parametrize("model_class, parameters", MODELS_AND_DATASETS)
|
||||||
@@ -89,7 +89,7 @@ def test_client_server_sklearn_inference(
|
|||||||
|
|
||||||
if get_model_name(model_class) == "KNeighborsClassifier":
|
if get_model_name(model_class) == "KNeighborsClassifier":
|
||||||
# Skipping KNN for this test
|
# Skipping KNN for this test
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4014
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4014
|
||||||
pytest.skip("Skipping KNN, because FHE predictions and clear ones are differents.")
|
pytest.skip("Skipping KNN, because FHE predictions and clear ones are differents.")
|
||||||
|
|
||||||
# Generate random data
|
# Generate random data
|
||||||
@@ -167,7 +167,7 @@ def test_client_server_sklearn_inference(
|
|||||||
|
|
||||||
|
|
||||||
# This is a known flaky test
|
# This is a known flaky test
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4014
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4014
|
||||||
@pytest.mark.flaky
|
@pytest.mark.flaky
|
||||||
@pytest.mark.parametrize("model_class, parameters", MODELS_AND_DATASETS)
|
@pytest.mark.parametrize("model_class, parameters", MODELS_AND_DATASETS)
|
||||||
@pytest.mark.parametrize("n_bits", [8])
|
@pytest.mark.parametrize("n_bits", [8])
|
||||||
@@ -912,7 +912,7 @@ def test_client_server_sklearn_training(
|
|||||||
# Fit the model with the created dataset to compile it for production
|
# Fit the model with the created dataset to compile it for production
|
||||||
# This step ensures the model knows the number of features, targets and features distribution
|
# This step ensures the model knows the number of features, targets and features distribution
|
||||||
# Remove this once this step is improved
|
# Remove this once this step is improved
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4466
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4466
|
||||||
model.fit(x_compile_set, y_compile_set, fhe="disable")
|
model.fit(x_compile_set, y_compile_set, fhe="disable")
|
||||||
|
|
||||||
# Check that client and server files are properly generated
|
# Check that client and server files are properly generated
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from concrete.ml.onnx.convert import OPSET_VERSION_FOR_ONNX_EXPORT
|
|||||||
from concrete.ml.onnx.onnx_utils import check_onnx_model
|
from concrete.ml.onnx.onnx_utils import check_onnx_model
|
||||||
|
|
||||||
|
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4604
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4604
|
||||||
@pytest.mark.skip()
|
@pytest.mark.skip()
|
||||||
def test_check_onnx_model_large():
|
def test_check_onnx_model_large():
|
||||||
"""Test that check_onnx_model can handle models larger than 2GB."""
|
"""Test that check_onnx_model can handle models larger than 2GB."""
|
||||||
|
|||||||
@@ -73,7 +73,7 @@ def generate_pandas_dataframe(
|
|||||||
|
|
||||||
# Make sure 0 is not included in the index
|
# Make sure 0 is not included in the index
|
||||||
# Remove this once NaN values are not represented by 0 anymore
|
# Remove this once NaN values are not represented by 0 anymore
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
if isinstance(indexes, int):
|
if isinstance(indexes, int):
|
||||||
indexes = list(range(1, indexes + 1))
|
indexes = list(range(1, indexes + 1))
|
||||||
|
|
||||||
@@ -218,7 +218,7 @@ def test_merge(as_method, how, selected_column):
|
|||||||
), "Joined encrypted data-frames decrypted by different clients are not equal."
|
), "Joined encrypted data-frames decrypted by different clients are not equal."
|
||||||
|
|
||||||
# Improve the test to avoid risk of flaky
|
# Improve the test to avoid risk of flaky
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
assert pandas_dataframe_are_equal(
|
assert pandas_dataframe_are_equal(
|
||||||
clear_df_joined_1, pandas_joined_df, float_atol=1, equal_nan=True
|
clear_df_joined_1, pandas_joined_df, float_atol=1, equal_nan=True
|
||||||
), "Joined encrypted data-frame does not match Pandas' joined data-frame."
|
), "Joined encrypted data-frame does not match Pandas' joined data-frame."
|
||||||
@@ -238,7 +238,7 @@ def test_pre_post_processing(dtype):
|
|||||||
clear_df = client.decrypt_to_pandas(encrypted_df)
|
clear_df = client.decrypt_to_pandas(encrypted_df)
|
||||||
|
|
||||||
# Improve the test to avoid risk of flaky
|
# Improve the test to avoid risk of flaky
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
assert pandas_dataframe_are_equal(
|
assert pandas_dataframe_are_equal(
|
||||||
pandas_df, clear_df, float_atol=1, equal_nan=include_nan
|
pandas_df, clear_df, float_atol=1, equal_nan=include_nan
|
||||||
), "Processed encrypted data-frame does not match Pandas' initial data-frame."
|
), "Processed encrypted data-frame does not match Pandas' initial data-frame."
|
||||||
@@ -263,7 +263,7 @@ def test_quantization_corner_cases(float_min_max):
|
|||||||
clear_df = client.decrypt_to_pandas(encrypted_df)
|
clear_df = client.decrypt_to_pandas(encrypted_df)
|
||||||
|
|
||||||
# Improve the test to avoid risk of flaky
|
# Improve the test to avoid risk of flaky
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
assert pandas_dataframe_are_equal(
|
assert pandas_dataframe_are_equal(
|
||||||
pandas_df, clear_df, float_atol=1, equal_nan=True
|
pandas_df, clear_df, float_atol=1, equal_nan=True
|
||||||
), "Processed encrypted data-frame does not match Pandas' initial data-frame."
|
), "Processed encrypted data-frame does not match Pandas' initial data-frame."
|
||||||
@@ -303,7 +303,7 @@ def test_save_load():
|
|||||||
loaded_clear_df = client.decrypt_to_pandas(loaded_encrypted_df)
|
loaded_clear_df = client.decrypt_to_pandas(loaded_encrypted_df)
|
||||||
|
|
||||||
# Improve the test to avoid risk of flaky
|
# Improve the test to avoid risk of flaky
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
assert pandas_dataframe_are_equal(
|
assert pandas_dataframe_are_equal(
|
||||||
loaded_clear_df, pandas_df, float_atol=1, equal_nan=True
|
loaded_clear_df, pandas_df, float_atol=1, equal_nan=True
|
||||||
), "Loaded encrypted data-frame does not match the initial encrypted data-frame."
|
), "Loaded encrypted data-frame does not match the initial encrypted data-frame."
|
||||||
@@ -540,7 +540,7 @@ def concrete_client_files_are_equal(
|
|||||||
|
|
||||||
|
|
||||||
# Improve this test if Concrete Python provides an official way to check such compatibility
|
# Improve this test if Concrete Python provides an official way to check such compatibility
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
def test_parameter_sets():
|
def test_parameter_sets():
|
||||||
"""Test if new generated parameter sets (client.zip) are equal to the ones stored in source."""
|
"""Test if new generated parameter sets (client.zip) are equal to the ones stored in source."""
|
||||||
with tempfile.TemporaryDirectory() as temp_dir:
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||||||
@@ -653,7 +653,7 @@ def test_schema_input():
|
|||||||
), "Joined encrypted data-frames decrypted by different clients are not equal."
|
), "Joined encrypted data-frames decrypted by different clients are not equal."
|
||||||
|
|
||||||
# Improve the test to avoid risk of flaky
|
# Improve the test to avoid risk of flaky
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4342
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4342
|
||||||
assert pandas_dataframe_are_equal(
|
assert pandas_dataframe_are_equal(
|
||||||
clear_df_joined_1, pandas_joined_df, float_atol=1, equal_nan=True
|
clear_df_joined_1, pandas_joined_df, float_atol=1, equal_nan=True
|
||||||
), "Joined encrypted data-frame does not match Pandas' joined data-frame."
|
), "Joined encrypted data-frame does not match Pandas' joined data-frame."
|
||||||
|
|||||||
@@ -301,9 +301,9 @@ def test_binary_search_for_built_in_models(model_class, parameters, threshold, p
|
|||||||
# NeuralNetRegressor models support a `predict_proba` method since it directly inherits from
|
# NeuralNetRegressor models support a `predict_proba` method since it directly inherits from
|
||||||
# Skorch but since Scikit-Learn does not, we don't as well. This issue could be fixed by making
|
# Skorch but since Scikit-Learn does not, we don't as well. This issue could be fixed by making
|
||||||
# neural networks not inherit from Skorch.
|
# neural networks not inherit from Skorch.
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||||
# Skipping predict_proba for KNN, doesn't work for now.
|
# Skipping predict_proba for KNN, doesn't work for now.
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||||
|
|
||||||
if predict == "predict_proba" and get_model_name(model_class) in [
|
if predict == "predict_proba" and get_model_name(model_class) in [
|
||||||
"NeuralNetRegressor",
|
"NeuralNetRegressor",
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ from concrete.ml.torch.numpy_module import NumpyModule
|
|||||||
INPUT_OUTPUT_FEATURE = [1, 2, 3]
|
INPUT_OUTPUT_FEATURE = [1, 2, 3]
|
||||||
|
|
||||||
|
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4172
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4172
|
||||||
@pytest.mark.flaky
|
@pytest.mark.flaky
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"model",
|
"model",
|
||||||
|
|||||||
@@ -56,7 +56,7 @@ N_BITS_LIST = [
|
|||||||
pytest.param(nn.GELU, id="GELU"),
|
pytest.param(nn.GELU, id="GELU"),
|
||||||
pytest.param(nn.LogSigmoid, id="LogSigmoid"),
|
pytest.param(nn.LogSigmoid, id="LogSigmoid"),
|
||||||
# Some issues are still encountered with some activations
|
# Some issues are still encountered with some activations
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/335
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/335
|
||||||
#
|
#
|
||||||
# Other problems, certainly related to tests:
|
# Other problems, certainly related to tests:
|
||||||
# Required positional arguments: 'embed_dim' and 'num_heads' and fails with a partial
|
# Required positional arguments: 'embed_dim' and 'num_heads' and fails with a partial
|
||||||
@@ -352,11 +352,11 @@ def test_quantized_module_rounding_fhe(model_class, input_shape, default_configu
|
|||||||
# Execute the model with rounding in FHE execution mode
|
# Execute the model with rounding in FHE execution mode
|
||||||
quantized_model.forward(numpy_test, fhe="execute")
|
quantized_model.forward(numpy_test, fhe="execute")
|
||||||
|
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3800
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3800
|
||||||
|
|
||||||
|
|
||||||
# Extend this test with multi-input encryption status
|
# Extend this test with multi-input encryption status
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4011
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4011
|
||||||
@pytest.mark.parametrize("model_class, input_shape", [pytest.param(FC, (100, 32 * 32 * 3))])
|
@pytest.mark.parametrize("model_class, input_shape", [pytest.param(FC, (100, 32 * 32 * 3))])
|
||||||
def test_inputs_encryption_status(model_class, input_shape, default_configuration):
|
def test_inputs_encryption_status(model_class, input_shape, default_configuration):
|
||||||
"""Check that giving inputs_encryption_status work properly."""
|
"""Check that giving inputs_encryption_status work properly."""
|
||||||
|
|||||||
@@ -201,7 +201,7 @@ def test_univariate_ops_no_attrs(
|
|||||||
|
|
||||||
|
|
||||||
# Manage ranges/improve tests for exponential
|
# Manage ranges/improve tests for exponential
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/229
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/229
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"n_bits",
|
"n_bits",
|
||||||
[pytest.param(n_bits) for n_bits in N_BITS_LIST],
|
[pytest.param(n_bits) for n_bits in N_BITS_LIST],
|
||||||
@@ -1685,7 +1685,7 @@ def test_brevitas_quant(check_r2_score, is_signed: bool, narrow: bool):
|
|||||||
)
|
)
|
||||||
|
|
||||||
if not is_signed and narrow:
|
if not is_signed and narrow:
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4544
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4544
|
||||||
# Reinstate warning check when brevitas export is fixed
|
# Reinstate warning check when brevitas export is fixed
|
||||||
pytest.skip("Skipping checking of invalid brevitas quant setting (signed=0,narrow=1)")
|
pytest.skip("Skipping checking of invalid brevitas quant setting (signed=0,narrow=1)")
|
||||||
# with pytest.raises(AssertionError, match=r"Can not use narrow range.*"):
|
# with pytest.raises(AssertionError, match=r"Can not use narrow range.*"):
|
||||||
|
|||||||
@@ -416,7 +416,7 @@ def check_onnx_file_dump(
|
|||||||
|
|
||||||
if model_name == "KNeighborsClassifier":
|
if model_name == "KNeighborsClassifier":
|
||||||
# KNN can only be compiled with small quantization bit numbers for now
|
# KNN can only be compiled with small quantization bit numbers for now
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3979
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3979
|
||||||
model.n_bits = 2
|
model.n_bits = 2
|
||||||
|
|
||||||
model.fit(x, y)
|
model.fit(x, y)
|
||||||
@@ -446,7 +446,7 @@ def check_onnx_file_dump(
|
|||||||
print(f"\nExpected {model_name=}:\n{str_expected}")
|
print(f"\nExpected {model_name=}:\n{str_expected}")
|
||||||
|
|
||||||
# Test equality when it does not depend on seeds
|
# Test equality when it does not depend on seeds
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3266
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3266
|
||||||
if not is_model_class_in_a_list(model_class, _get_sklearn_tree_models(select="RandomForest")):
|
if not is_model_class_in_a_list(model_class, _get_sklearn_tree_models(select="RandomForest")):
|
||||||
# The expected graph is usually a string and we therefore directly test if it is equal to
|
# The expected graph is usually a string and we therefore directly test if it is equal to
|
||||||
# the retrieved graph's string. However, in some cases such as for TweedieRegressor models,
|
# the retrieved graph's string. However, in some cases such as for TweedieRegressor models,
|
||||||
|
|||||||
@@ -164,7 +164,7 @@ def get_n_bits_non_correctness(model_class):
|
|||||||
"""Get the number of bits to use for non correctness related tests."""
|
"""Get the number of bits to use for non correctness related tests."""
|
||||||
|
|
||||||
# KNN can only be compiled with small quantization bit numbers for now
|
# KNN can only be compiled with small quantization bit numbers for now
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3979
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3979
|
||||||
if get_model_name(model_class) == "KNeighborsClassifier":
|
if get_model_name(model_class) == "KNeighborsClassifier":
|
||||||
n_bits = 2
|
n_bits = 2
|
||||||
else:
|
else:
|
||||||
@@ -206,7 +206,7 @@ def check_correctness_with_sklearn(
|
|||||||
|
|
||||||
# If the model is a classifier
|
# If the model is a classifier
|
||||||
# KNeighborsClassifier does not provide a predict_proba method for now
|
# KNeighborsClassifier does not provide a predict_proba method for now
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||||
if (
|
if (
|
||||||
is_classifier_or_partial_classifier(model)
|
is_classifier_or_partial_classifier(model)
|
||||||
and get_model_name(model_class) != "KNeighborsClassifier"
|
and get_model_name(model_class) != "KNeighborsClassifier"
|
||||||
@@ -263,7 +263,7 @@ def check_double_fit(model_class, n_bits, x_1, x_2, y_1, y_2):
|
|||||||
"""Check double fit."""
|
"""Check double fit."""
|
||||||
|
|
||||||
if get_model_name(model_class) == "KNeighborsClassifier":
|
if get_model_name(model_class) == "KNeighborsClassifier":
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4014
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4014
|
||||||
pytest.skip(
|
pytest.skip(
|
||||||
"Given that KNN is not accurate and the test data-set is small"
|
"Given that KNN is not accurate and the test data-set is small"
|
||||||
"the y_pred1 and y_pred2 can be equal."
|
"the y_pred1 and y_pred2 can be equal."
|
||||||
@@ -420,7 +420,7 @@ def check_serialization_dump_load(model, x, use_dump_method):
|
|||||||
assert numpy.array_equal(y_pred_sklearn_model, y_pred_loaded_sklearn_model)
|
assert numpy.array_equal(y_pred_sklearn_model, y_pred_loaded_sklearn_model)
|
||||||
|
|
||||||
# Add a test to check that graphs before and after the serialization are identical
|
# Add a test to check that graphs before and after the serialization are identical
|
||||||
# FIME: https://github.com/zama-ai/concrete-ml-internal/issues/4175
|
# FIME: https://github.com/luxfi/concrete-ml-internal/issues/4175
|
||||||
|
|
||||||
|
|
||||||
def check_serialization_dumps_loads(model, x, use_dump_method):
|
def check_serialization_dumps_loads(model, x, use_dump_method):
|
||||||
@@ -476,7 +476,7 @@ def check_serialization_dumps_loads(model, x, use_dump_method):
|
|||||||
assert numpy.array_equal(y_pred_sklearn_model, y_pred_loaded_sklearn_model)
|
assert numpy.array_equal(y_pred_sklearn_model, y_pred_loaded_sklearn_model)
|
||||||
|
|
||||||
# Add a test to check that graphs before and after the serialization are identical
|
# Add a test to check that graphs before and after the serialization are identical
|
||||||
# FIME: https://github.com/zama-ai/concrete-ml-internal/issues/4175
|
# FIME: https://github.com/luxfi/concrete-ml-internal/issues/4175
|
||||||
|
|
||||||
|
|
||||||
def check_offset(model_class, n_bits, x, y):
|
def check_offset(model_class, n_bits, x, y):
|
||||||
@@ -502,7 +502,7 @@ def check_inference_methods(model, model_class, x, check_float_array_equal):
|
|||||||
# method accessible by anyone, without having any FHE implementation. As this could create some
|
# method accessible by anyone, without having any FHE implementation. As this could create some
|
||||||
# confusion, a NotImplementedError is raised. This issue could be fixed by making these classes
|
# confusion, a NotImplementedError is raised. This issue could be fixed by making these classes
|
||||||
# not inherit from skorch.
|
# not inherit from skorch.
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3373
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3373
|
||||||
if get_model_name(model) == "NeuralNetRegressor":
|
if get_model_name(model) == "NeuralNetRegressor":
|
||||||
with pytest.raises(
|
with pytest.raises(
|
||||||
NotImplementedError,
|
NotImplementedError,
|
||||||
@@ -514,7 +514,7 @@ def check_inference_methods(model, model_class, x, check_float_array_equal):
|
|||||||
model.predict_proba(x)
|
model.predict_proba(x)
|
||||||
|
|
||||||
# KNeighborsClassifier does not provide a predict_proba method for now
|
# KNeighborsClassifier does not provide a predict_proba method for now
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||||
elif get_model_name(model) == "KNeighborsClassifier":
|
elif get_model_name(model) == "KNeighborsClassifier":
|
||||||
with pytest.raises(
|
with pytest.raises(
|
||||||
NotImplementedError,
|
NotImplementedError,
|
||||||
@@ -526,7 +526,7 @@ def check_inference_methods(model, model_class, x, check_float_array_equal):
|
|||||||
model.predict_proba(x)
|
model.predict_proba(x)
|
||||||
|
|
||||||
# KNeighborsClassifier does not provide a kneighbors method
|
# KNeighborsClassifier does not provide a kneighbors method
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4080
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4080
|
||||||
with pytest.raises(
|
with pytest.raises(
|
||||||
NotImplementedError,
|
NotImplementedError,
|
||||||
match=(
|
match=(
|
||||||
@@ -628,7 +628,7 @@ def check_separated_inference(model, fhe_circuit, x, check_float_array_equal):
|
|||||||
y_pred = model.post_processing(y_pred)
|
y_pred = model.post_processing(y_pred)
|
||||||
|
|
||||||
# KNeighborsClassifier does not provide a predict_proba method for now
|
# KNeighborsClassifier does not provide a predict_proba method for now
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||||
if (
|
if (
|
||||||
is_classifier_or_partial_classifier(model)
|
is_classifier_or_partial_classifier(model)
|
||||||
and get_model_name(model) != "KNeighborsClassifier"
|
and get_model_name(model) != "KNeighborsClassifier"
|
||||||
@@ -705,7 +705,7 @@ def check_input_support(model_class, n_bits, default_configuration, x, y, input_
|
|||||||
|
|
||||||
# Similarly, we test `predict_proba` for classifiers
|
# Similarly, we test `predict_proba` for classifiers
|
||||||
# KNeighborsClassifier does not provide a predict_proba method for now
|
# KNeighborsClassifier does not provide a predict_proba method for now
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||||
if (
|
if (
|
||||||
is_classifier_or_partial_classifier(model)
|
is_classifier_or_partial_classifier(model)
|
||||||
and get_model_name(model_class) != "KNeighborsClassifier"
|
and get_model_name(model_class) != "KNeighborsClassifier"
|
||||||
@@ -719,7 +719,7 @@ def check_input_support(model_class, n_bits, default_configuration, x, y, input_
|
|||||||
|
|
||||||
# Similarly, we test `predict_proba` for classifiers
|
# Similarly, we test `predict_proba` for classifiers
|
||||||
# KNeighborsClassifier does not provide a predict_proba method for now
|
# KNeighborsClassifier does not provide a predict_proba method for now
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||||
if (
|
if (
|
||||||
is_classifier_or_partial_classifier(model)
|
is_classifier_or_partial_classifier(model)
|
||||||
and get_model_name(model_class) != "KNeighborsClassifier"
|
and get_model_name(model_class) != "KNeighborsClassifier"
|
||||||
@@ -797,7 +797,7 @@ def check_grid_search(model_class, x, y, scoring):
|
|||||||
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
|
warnings.simplefilter("ignore", category=UndefinedMetricWarning)
|
||||||
|
|
||||||
# KNeighborsClassifier does not provide a predict_proba method for now
|
# KNeighborsClassifier does not provide a predict_proba method for now
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||||
if get_model_name(model_class) == "KNeighborsClassifier" and scoring in [
|
if get_model_name(model_class) == "KNeighborsClassifier" and scoring in [
|
||||||
"roc_auc",
|
"roc_auc",
|
||||||
"average_precision",
|
"average_precision",
|
||||||
@@ -932,7 +932,7 @@ def check_fitted_compiled_error_raises(model_class, n_bits, x, y):
|
|||||||
model.predict(x)
|
model.predict(x)
|
||||||
|
|
||||||
# KNeighborsClassifier does not provide a predict_proba method for now
|
# KNeighborsClassifier does not provide a predict_proba method for now
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||||
if (
|
if (
|
||||||
is_classifier_or_partial_classifier(model_class)
|
is_classifier_or_partial_classifier(model_class)
|
||||||
and get_model_name(model) != "KNeighborsClassifier"
|
and get_model_name(model) != "KNeighborsClassifier"
|
||||||
@@ -1384,7 +1384,7 @@ def check_rounding_consistency(
|
|||||||
|
|
||||||
# Check that the maximum bit-width of the circuit with rounding is at most:
|
# Check that the maximum bit-width of the circuit with rounding is at most:
|
||||||
# maximum bit-width (of the circuit without rounding) + 2
|
# maximum bit-width (of the circuit without rounding) + 2
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4178
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4178
|
||||||
|
|
||||||
|
|
||||||
def check_sum_for_tree_based_models(
|
def check_sum_for_tree_based_models(
|
||||||
@@ -1706,7 +1706,7 @@ def test_inference_methods(
|
|||||||
|
|
||||||
# Pipeline test sometimes fails with RandomForest models. This bug may come from Hummingbird
|
# Pipeline test sometimes fails with RandomForest models. This bug may come from Hummingbird
|
||||||
# and needs further investigations
|
# and needs further investigations
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2779
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2779
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"model_class, parameters",
|
"model_class, parameters",
|
||||||
get_sklearn_all_models_and_datasets(ignore="RandomForest"),
|
get_sklearn_all_models_and_datasets(ignore="RandomForest"),
|
||||||
@@ -1773,7 +1773,7 @@ def test_predict_correctness(
|
|||||||
"""Test prediction correctness between clear quantized and FHE simulation or execution."""
|
"""Test prediction correctness between clear quantized and FHE simulation or execution."""
|
||||||
|
|
||||||
# KNN can only be compiled with small quantization bit numbers for now
|
# KNN can only be compiled with small quantization bit numbers for now
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3979
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3979
|
||||||
if n_bits > 5 and get_model_name(model_class) == "KNeighborsClassifier":
|
if n_bits > 5 and get_model_name(model_class) == "KNeighborsClassifier":
|
||||||
pytest.skip("KNeighborsClassifier models can only run with 5 bits at most.")
|
pytest.skip("KNeighborsClassifier models can only run with 5 bits at most.")
|
||||||
|
|
||||||
@@ -1817,7 +1817,7 @@ def test_separated_inference(
|
|||||||
n_bits = min(N_BITS_REGULAR_BUILDS)
|
n_bits = min(N_BITS_REGULAR_BUILDS)
|
||||||
|
|
||||||
# KNN can only be compiled with small quantization bit numbers for now
|
# KNN can only be compiled with small quantization bit numbers for now
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3979
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3979
|
||||||
if n_bits > 5 and get_model_name(model_class) == "KNeighborsClassifier":
|
if n_bits > 5 and get_model_name(model_class) == "KNeighborsClassifier":
|
||||||
pytest.skip("KNeighborsClassifier models can only run with 5 bits at most.")
|
pytest.skip("KNeighborsClassifier models can only run with 5 bits at most.")
|
||||||
|
|
||||||
@@ -1862,7 +1862,7 @@ def test_fitted_compiled_error_raises(
|
|||||||
check_fitted_compiled_error_raises(model_class, n_bits, x, y)
|
check_fitted_compiled_error_raises(model_class, n_bits, x, y)
|
||||||
|
|
||||||
|
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4169
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4169
|
||||||
@pytest.mark.flaky
|
@pytest.mark.flaky
|
||||||
@pytest.mark.parametrize("model_class, parameters", MODELS_AND_DATASETS)
|
@pytest.mark.parametrize("model_class, parameters", MODELS_AND_DATASETS)
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
@@ -1898,7 +1898,7 @@ def test_p_error_simulation(
|
|||||||
"""Detect divergence between simulated/FHE execution and clear run."""
|
"""Detect divergence between simulated/FHE execution and clear run."""
|
||||||
|
|
||||||
# KNeighborsClassifier does not provide a predict_proba method for now
|
# KNeighborsClassifier does not provide a predict_proba method for now
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3962
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3962
|
||||||
predict_function = (
|
predict_function = (
|
||||||
model.predict_proba
|
model.predict_proba
|
||||||
if is_classifier_or_partial_classifier(model)
|
if is_classifier_or_partial_classifier(model)
|
||||||
@@ -1921,7 +1921,7 @@ def test_p_error_simulation(
|
|||||||
|
|
||||||
# Skip the following if model is linear
|
# Skip the following if model is linear
|
||||||
# Simulation and FHE differs with very high p_error on leveled circuit
|
# Simulation and FHE differs with very high p_error on leveled circuit
|
||||||
# FIXME https://github.com/zama-ai/concrete-ml-internal/issues/4343
|
# FIXME https://github.com/luxfi/concrete-ml-internal/issues/4343
|
||||||
if is_linear_model:
|
if is_linear_model:
|
||||||
pytest.skip("Skipping test for linear models")
|
pytest.skip("Skipping test for linear models")
|
||||||
|
|
||||||
@@ -2063,7 +2063,7 @@ def test_linear_models_have_no_tlu(
|
|||||||
|
|
||||||
# This test does not check rounding at level 2
|
# This test does not check rounding at level 2
|
||||||
# Additional tests for this purpose should be added in future updates
|
# Additional tests for this purpose should be added in future updates
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4179
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4179
|
||||||
@pytest.mark.parametrize("model_class, parameters", get_sklearn_tree_models_and_datasets())
|
@pytest.mark.parametrize("model_class, parameters", get_sklearn_tree_models_and_datasets())
|
||||||
@pytest.mark.parametrize("n_bits", [2, 5, 8])
|
@pytest.mark.parametrize("n_bits", [2, 5, 8])
|
||||||
def test_rounding_consistency_for_regular_models(
|
def test_rounding_consistency_for_regular_models(
|
||||||
@@ -2139,7 +2139,7 @@ def test_sum_for_tree_based_models(
|
|||||||
|
|
||||||
|
|
||||||
# This test should be extended to all built-in models.
|
# This test should be extended to all built-in models.
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4234
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4234
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"n_bits, error_message",
|
"n_bits, error_message",
|
||||||
[
|
[
|
||||||
@@ -2168,7 +2168,7 @@ def test_invalid_n_bits_setting(model_class, n_bits, error_message):
|
|||||||
|
|
||||||
|
|
||||||
# This test should be extended to all built-in models.
|
# This test should be extended to all built-in models.
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4234
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4234
|
||||||
@pytest.mark.parametrize("n_bits", [5, {"op_inputs": 5}, {"op_inputs": 2, "op_leaves": 1}])
|
@pytest.mark.parametrize("n_bits", [5, {"op_inputs": 5}, {"op_inputs": 2, "op_leaves": 1}])
|
||||||
@pytest.mark.parametrize("model_class, parameters", get_sklearn_tree_models_and_datasets())
|
@pytest.mark.parametrize("model_class, parameters", get_sklearn_tree_models_and_datasets())
|
||||||
def test_valid_n_bits_setting(
|
def test_valid_n_bits_setting(
|
||||||
@@ -2230,7 +2230,7 @@ def test_error_raise_unsupported_pandas_values(model_class, bad_value, expected_
|
|||||||
|
|
||||||
|
|
||||||
# Add QNNs in this test
|
# Add QNNs in this test
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4436
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4436
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"model_class, parameters",
|
"model_class, parameters",
|
||||||
get_sklearn_linear_models_and_datasets()
|
get_sklearn_linear_models_and_datasets()
|
||||||
|
|||||||
@@ -167,7 +167,7 @@ def train_brevitas_network_tinymnist(is_cnn, qat_bits, signed, narrow, pot_scali
|
|||||||
|
|
||||||
|
|
||||||
# This test is a known flaky
|
# This test is a known flaky
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3933
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3933
|
||||||
@pytest.mark.flaky
|
@pytest.mark.flaky
|
||||||
@pytest.mark.parametrize("qat_bits", [3])
|
@pytest.mark.parametrize("qat_bits", [3])
|
||||||
@pytest.mark.parametrize("signed, narrow", [(True, False), (True, True), (False, False)])
|
@pytest.mark.parametrize("signed, narrow", [(True, False), (True, True), (False, False)])
|
||||||
@@ -234,7 +234,7 @@ def test_brevitas_tinymnist_cnn(
|
|||||||
# Accept, at most, 1% examples that are classified differently (currently 5)
|
# Accept, at most, 1% examples that are classified differently (currently 5)
|
||||||
# For now, the correctness test has been disabled as it was too flaky, it should however be put
|
# For now, the correctness test has been disabled as it was too flaky, it should however be put
|
||||||
# back at one point
|
# back at one point
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2550
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2550
|
||||||
# assert abs(fhe_simulation_correct - torch_correct) <= numpy.ceil(0.01 * len(y_test))
|
# assert abs(fhe_simulation_correct - torch_correct) <= numpy.ceil(0.01 * len(y_test))
|
||||||
|
|
||||||
assert fhe_s_correct >= 0
|
assert fhe_s_correct >= 0
|
||||||
@@ -245,7 +245,7 @@ def test_brevitas_tinymnist_cnn(
|
|||||||
|
|
||||||
# Note that this test is currently disabled until the pytorch dtype issue is found
|
# Note that this test is currently disabled until the pytorch dtype issue is found
|
||||||
# and all mismatches between Concrete ML and Brevitas are fixed
|
# and all mismatches between Concrete ML and Brevitas are fixed
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2373
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2373
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"n_layers",
|
"n_layers",
|
||||||
[3],
|
[3],
|
||||||
@@ -496,7 +496,7 @@ def test_brevitas_constant_folding(default_configuration):
|
|||||||
|
|
||||||
|
|
||||||
# This test is a known flaky
|
# This test is a known flaky
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4356
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4356
|
||||||
@pytest.mark.flaky
|
@pytest.mark.flaky
|
||||||
@pytest.mark.parametrize("manual_rounding", [None, 3])
|
@pytest.mark.parametrize("manual_rounding", [None, 3])
|
||||||
@pytest.mark.parametrize("power_of_two", [True, False])
|
@pytest.mark.parametrize("power_of_two", [True, False])
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ def compile_and_test_keras(
|
|||||||
|
|
||||||
|
|
||||||
# We should also have some correctness tests
|
# We should also have some correctness tests
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/2749
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/2749
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
|
|||||||
@@ -398,13 +398,13 @@ def accuracy_test_rounding(
|
|||||||
for key, module in compiled_modules.items():
|
for key, module in compiled_modules.items():
|
||||||
|
|
||||||
# low bit-width rounding is not behaving as expected with new simulation
|
# low bit-width rounding is not behaving as expected with new simulation
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4331
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4331
|
||||||
if "low" not in key:
|
if "low" not in key:
|
||||||
check_is_good_execution_for_cml_vs_circuit(x_test, module, simulate=simulate)
|
check_is_good_execution_for_cml_vs_circuit(x_test, module, simulate=simulate)
|
||||||
|
|
||||||
# FIXME: The following MSE comparison is commented out due to instability issues.
|
# FIXME: The following MSE comparison is commented out due to instability issues.
|
||||||
# We will investigate a better way to assess the rounding feature's performance.
|
# We will investigate a better way to assess the rounding feature's performance.
|
||||||
# https://github.com/zama-ai/concrete-ml-internal/issues/3662
|
# https://github.com/luxfi/concrete-ml-internal/issues/3662
|
||||||
# mse_results = {
|
# mse_results = {
|
||||||
# key: numpy.mean(numpy.square(numpy.subtract(results['original'], result_list)))
|
# key: numpy.mean(numpy.square(numpy.subtract(results['original'], result_list)))
|
||||||
# for key, result_list in results.items()
|
# for key, result_list in results.items()
|
||||||
@@ -416,7 +416,7 @@ def accuracy_test_rounding(
|
|||||||
|
|
||||||
|
|
||||||
# This test is a known flaky
|
# This test is a known flaky
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3429
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3429
|
||||||
@pytest.mark.flaky
|
@pytest.mark.flaky
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"activation_function",
|
"activation_function",
|
||||||
@@ -481,7 +481,7 @@ def test_compile_torch_or_onnx_networks(
|
|||||||
|
|
||||||
|
|
||||||
# This test is a known flaky
|
# This test is a known flaky
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3660
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3660
|
||||||
@pytest.mark.flaky
|
@pytest.mark.flaky
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"activation_function",
|
"activation_function",
|
||||||
@@ -564,7 +564,7 @@ def test_compile_torch_or_onnx_conv_networks( # pylint: disable=unused-argument
|
|||||||
pytest.param(nn.GELU, id="GELU"),
|
pytest.param(nn.GELU, id="GELU"),
|
||||||
pytest.param(nn.LogSigmoid, id="LogSigmoid"),
|
pytest.param(nn.LogSigmoid, id="LogSigmoid"),
|
||||||
# Some issues are still encountered with some activations
|
# Some issues are still encountered with some activations
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/335
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/335
|
||||||
#
|
#
|
||||||
# Other problems, certainly related to tests:
|
# Other problems, certainly related to tests:
|
||||||
# Required positional arguments: 'embed_dim' and 'num_heads' and fails with a partial
|
# Required positional arguments: 'embed_dim' and 'num_heads' and fails with a partial
|
||||||
@@ -699,7 +699,7 @@ def test_compile_brevitas_qat(
|
|||||||
|
|
||||||
|
|
||||||
# Update this test to align with Concrete's simulation fix.
|
# Update this test to align with Concrete's simulation fix.
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4578
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4578
|
||||||
@pytest.mark.xfail
|
@pytest.mark.xfail
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"model_class, expected_onnx_str",
|
"model_class, expected_onnx_str",
|
||||||
@@ -1031,7 +1031,7 @@ def test_shape_operations_net(
|
|||||||
model = model_class(is_qat)
|
model = model_class(is_qat)
|
||||||
|
|
||||||
# Shape transformation do not support >1 example in the inputset
|
# Shape transformation do not support >1 example in the inputset
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3871
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3871
|
||||||
inputset = numpy.random.uniform(size=(1, n_channels, 2, 2))
|
inputset = numpy.random.uniform(size=(1, n_channels, 2, 2))
|
||||||
|
|
||||||
if is_qat:
|
if is_qat:
|
||||||
|
|||||||
@@ -62,7 +62,7 @@ from concrete.ml.torch import NumpyModule
|
|||||||
pytest.param(nn.LogSigmoid, id="LogSigmoid"),
|
pytest.param(nn.LogSigmoid, id="LogSigmoid"),
|
||||||
pytest.param(nn.GELU, id="GELU"),
|
pytest.param(nn.GELU, id="GELU"),
|
||||||
# Some issues are still encountered with some activations
|
# Some issues are still encountered with some activations
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/335
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/335
|
||||||
#
|
#
|
||||||
# Other problems, certainly related to tests:
|
# Other problems, certainly related to tests:
|
||||||
# Required positional arguments: 'embed_dim' and 'num_heads' and fails with a partial
|
# Required positional arguments: 'embed_dim' and 'num_heads' and fails with a partial
|
||||||
|
|||||||
@@ -110,7 +110,7 @@ def test_sgd_training_manual(
|
|||||||
): # pylint: disable=unused-argument
|
): # pylint: disable=unused-argument
|
||||||
"""Trains a logistic regression with SGD in torch and quantized."""
|
"""Trains a logistic regression with SGD in torch and quantized."""
|
||||||
# Train on the bias when multi output is available in concrete
|
# Train on the bias when multi output is available in concrete
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4131
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/4131
|
||||||
|
|
||||||
print("test_sgd_training_manual", get_device)
|
print("test_sgd_training_manual", get_device)
|
||||||
|
|
||||||
|
|||||||
@@ -67,7 +67,7 @@ DATASETS_ARGS = {
|
|||||||
),
|
),
|
||||||
},
|
},
|
||||||
# Separate FMNIST from CIFAR directory
|
# Separate FMNIST from CIFAR directory
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3552
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3552
|
||||||
"FashionMNIST": {
|
"FashionMNIST": {
|
||||||
"dataset": datasets.FashionMNIST,
|
"dataset": datasets.FashionMNIST,
|
||||||
"mean": (0.2859),
|
"mean": (0.2859),
|
||||||
|
|||||||
+1
-1
@@ -21,7 +21,7 @@ KEYGEN_CACHE_DIR = CURRENT_DIR.joinpath(".keycache")
|
|||||||
|
|
||||||
# Add MPS (for macOS with Apple Silicon or AMD GPUs) support when error is fixed. For now, we
|
# Add MPS (for macOS with Apple Silicon or AMD GPUs) support when error is fixed. For now, we
|
||||||
# observe a decrease in torch's top1 accuracy when using MPS devices
|
# observe a decrease in torch's top1 accuracy when using MPS devices
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3953
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3953
|
||||||
|
|
||||||
# For PyTorch operations, we use CPU (simpler and avoids device mismatch issues)
|
# For PyTorch operations, we use CPU (simpler and avoids device mismatch issues)
|
||||||
DEVICE = "cpu"
|
DEVICE = "cpu"
|
||||||
|
|||||||
+1
-1
@@ -73,7 +73,7 @@ def main(args):
|
|||||||
|
|
||||||
# Add MPS (for macOS with Apple Silicon or AMD GPUs) support when error is fixed. For now, we
|
# Add MPS (for macOS with Apple Silicon or AMD GPUs) support when error is fixed. For now, we
|
||||||
# observe a decrease in torch's top1 accuracy when using MPS devices
|
# observe a decrease in torch's top1 accuracy when using MPS devices
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3953
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3953
|
||||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||||
compilation_device = "cuda" if concrete.compiler.check_gpu_available() else "cpu"
|
compilation_device = "cuda" if concrete.compiler.check_gpu_available() else "cpu"
|
||||||
|
|
||||||
|
|||||||
@@ -201,7 +201,7 @@ class Trainer(object):
|
|||||||
else:
|
else:
|
||||||
# Add MPS (for macOS with Apple Silicon or AMD GPUs) support when error is fixed. For
|
# Add MPS (for macOS with Apple Silicon or AMD GPUs) support when error is fixed. For
|
||||||
# now, we observe a decrease in torch's top1 accuracy when using MPS devices
|
# now, we observe a decrease in torch's top1 accuracy when using MPS devices
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3953
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3953
|
||||||
self.device = "cpu"
|
self.device = "cpu"
|
||||||
|
|
||||||
self.device = torch.device(self.device)
|
self.device = torch.device(self.device)
|
||||||
|
|||||||
@@ -54,7 +54,7 @@ def train(dev_folder="./dev"):
|
|||||||
|
|
||||||
# Add MPS (for macOS with Apple Silicon or AMD GPUs) support when error is fixed. For now, we
|
# Add MPS (for macOS with Apple Silicon or AMD GPUs) support when error is fixed. For now, we
|
||||||
# observe a decrease in torch's top1 accuracy when using MPS devices
|
# observe a decrease in torch's top1 accuracy when using MPS devices
|
||||||
# FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3953
|
# FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3953
|
||||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||||
|
|
||||||
# # Load the tokenizer (converts text to tokens)
|
# # Load the tokenizer (converts text to tokens)
|
||||||
@@ -167,7 +167,7 @@ def train(dev_folder="./dev"):
|
|||||||
print(f"Compilation time: {end - start:.4f} seconds")
|
print(f"Compilation time: {end - start:.4f} seconds")
|
||||||
|
|
||||||
# Write a custom example and predict in FHE
|
# Write a custom example and predict in FHE
|
||||||
tested_tweet = ["AirFrance is awesome, almost as much as Zama!"]
|
tested_tweet = ["AirFrance is awesome, almost as much as Lux!"]
|
||||||
X_tested_tweet = text_to_tensor(tested_tweet, transformer_model, tokenizer, device)
|
X_tested_tweet = text_to_tensor(tested_tweet, transformer_model, tokenizer, device)
|
||||||
clear_proba = best_model.predict_proba(X_tested_tweet)
|
clear_proba = best_model.predict_proba(X_tested_tweet)
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ It takes as input a folder with:
|
|||||||
It builds a Docker image and spawns a Docker container that runs the server.
|
It builds a Docker image and spawns a Docker container that runs the server.
|
||||||
|
|
||||||
This module is untested as it would require to first build the release Docker image.
|
This module is untested as it would require to first build the release Docker image.
|
||||||
FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3347
|
FIXME: https://github.com/luxfi/concrete-ml-internal/issues/3347
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
|||||||
@@ -425,7 +425,7 @@ class DualArray:
|
|||||||
|
|
||||||
# Concrete-Python does not support numpy.array_split and numpy.take so we need to build a custom
|
# Concrete-Python does not support numpy.array_split and numpy.take so we need to build a custom
|
||||||
# split method instead
|
# split method instead
|
||||||
# FIXME: https://github.com/zama-ai/concrete-internal/issues/329
|
# FIXME: https://github.com/luxfi/concrete-internal/issues/329
|
||||||
def enc_split(self, n: int, axis: int, key: str) -> Tuple[DualArray]:
|
def enc_split(self, n: int, axis: int, key: str) -> Tuple[DualArray]:
|
||||||
"""Split the arrays in n parts along a given axis."""
|
"""Split the arrays in n parts along a given axis."""
|
||||||
self_int_array = self._ensure_quantized(key=f"{key}_self")
|
self_int_array = self._ensure_quantized(key=f"{key}_self")
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
__name__ = "concrete-ml-extensions"
|
__name__ = "concrete-ml-extensions"
|
||||||
__author__ = "Zama"
|
__author__ = "Lux Network"
|
||||||
__all__ = ["concrete-ml-extensions"]
|
__all__ = ["concrete-ml-extensions"]
|
||||||
__version__ = "0.2.0"
|
__version__ = "0.2.0"
|
||||||
|
|
||||||
|
|||||||
Executable
+272
@@ -0,0 +1,272 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Migrate Zama FHE repos to Lux FHE stack
|
||||||
|
# Forks to github.com/luxfi, strips Zama references, updates imports
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
ZAMA_DIR="$HOME/work/zama"
|
||||||
|
LUX_FHE_DIR="$HOME/work/lux/fhe"
|
||||||
|
LUXFI_ORG="luxfi"
|
||||||
|
|
||||||
|
# Color output
|
||||||
|
RED='\033[0;31m'
|
||||||
|
GREEN='\033[0;32m'
|
||||||
|
YELLOW='\033[1;33m'
|
||||||
|
NC='\033[0m'
|
||||||
|
|
||||||
|
log() { echo -e "${GREEN}[+]${NC} $1"; }
|
||||||
|
warn() { echo -e "${YELLOW}[!]${NC} $1"; }
|
||||||
|
error() { echo -e "${RED}[-]${NC} $1"; }
|
||||||
|
|
||||||
|
# Repos to migrate with their new names
|
||||||
|
declare -A MIGRATE_REPOS=(
|
||||||
|
# Core libraries
|
||||||
|
["concrete-compiler-internal-llvm-project"]="fhe-compiler"
|
||||||
|
["concrete-ntt"]="fhe-ntt"
|
||||||
|
["concrete-fft"]="fhe-fft"
|
||||||
|
|
||||||
|
# ML
|
||||||
|
["concrete-ml"]="fhe-ml"
|
||||||
|
["concrete-ml-extensions"]="fhe-ml-extensions"
|
||||||
|
["concrete-ml-processing-rs"]="fhe-ml-rs"
|
||||||
|
|
||||||
|
# Templates & dApps
|
||||||
|
["dapps"]="fhe-dapps"
|
||||||
|
["fhevm-hardhat-template"]="fhe-hardhat-template"
|
||||||
|
["fhevm-next-template"]="fhe-next-template"
|
||||||
|
["fhevm-react-template"]="fhe-react-template"
|
||||||
|
["fhevm-vue-template"]="fhe-vue-template"
|
||||||
|
["fhevm-remix-plugin"]="fhe-remix-plugin"
|
||||||
|
["fhevm-workshop"]="fhe-workshop"
|
||||||
|
["fhevm-test-suite"]="fhe-test-suite"
|
||||||
|
["fhevm-mocks"]="fhe-mocks"
|
||||||
|
|
||||||
|
# Infrastructure
|
||||||
|
["fhevm-contracts"]="fhe-contracts"
|
||||||
|
["fhevm-go"]="fhe-go"
|
||||||
|
["fhevm-solidity"]="fhe-solidity"
|
||||||
|
["fhevm-decryptions-db"]="fhe-decryptions-db"
|
||||||
|
["fhevm-tfhe-cli"]="fhe-cli"
|
||||||
|
["go-ethereum-coprocessor"]="fhe-geth"
|
||||||
|
["relayer-sdk"]="fhe-relayer"
|
||||||
|
|
||||||
|
# Research & Docs
|
||||||
|
["tfhe-rs-handbook"]="fhe-handbook"
|
||||||
|
["verifiable-fhe-paper"]="fhe-papers"
|
||||||
|
["threshold-fhe"]="fhe-threshold"
|
||||||
|
|
||||||
|
# Demos
|
||||||
|
["fhe_ios_demo"]="fhe-ios"
|
||||||
|
["fhe-biometrics"]="fhe-biometrics"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Skip these (Zama-specific, already have, or third-party)
|
||||||
|
SKIP_REPOS=(
|
||||||
|
"tfhe-rs" # Use luxfi/fhe instead
|
||||||
|
"concrete" # Zama main product
|
||||||
|
"awesome-zama" # Promotional
|
||||||
|
"bounty-ecdsa-signature"
|
||||||
|
"bounty-matrix-inversion"
|
||||||
|
"bounty-program"
|
||||||
|
"ci-templates"
|
||||||
|
"fhenix" # Third party
|
||||||
|
"HElib" # IBM
|
||||||
|
"SEAL" # Microsoft
|
||||||
|
"openfhe-development"
|
||||||
|
"slab-github-runner"
|
||||||
|
"terraform-mpc-modules"
|
||||||
|
"tfhe-backward-compat-data"
|
||||||
|
"zbc-go-ethereum"
|
||||||
|
"hpu_fpga"
|
||||||
|
"hw_regmap"
|
||||||
|
"ocp-fhe"
|
||||||
|
"poc-prime-match"
|
||||||
|
"progress-tracker-python"
|
||||||
|
"copz25-code"
|
||||||
|
"blockchain-wallet-exporter"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Files/patterns to strip Zama references from
|
||||||
|
ZAMA_PATTERNS=(
|
||||||
|
"zama.ai"
|
||||||
|
"zama-ai"
|
||||||
|
"Zama"
|
||||||
|
"ZAMA"
|
||||||
|
"tfhe-rs"
|
||||||
|
"github.com/zama-ai"
|
||||||
|
"@zama-ai"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Replacement mappings
|
||||||
|
declare -A REPLACEMENTS=(
|
||||||
|
["zama.ai"]="lux.network"
|
||||||
|
["zama-ai"]="luxfi"
|
||||||
|
["Zama"]="Lux"
|
||||||
|
["ZAMA"]="LUX"
|
||||||
|
["tfhe-rs"]="luxfi/fhe"
|
||||||
|
["github.com/zama-ai"]="github.com/luxfi"
|
||||||
|
["@zama-ai"]="@luxfi"
|
||||||
|
["concrete-ml"]="fhe-ml"
|
||||||
|
["concrete-ntt"]="fhe-ntt"
|
||||||
|
["concrete-fft"]="fhe-fft"
|
||||||
|
["fhevm-"]="fhe-"
|
||||||
|
)
|
||||||
|
|
||||||
|
fork_repo() {
|
||||||
|
local src_name="$1"
|
||||||
|
local dst_name="$2"
|
||||||
|
|
||||||
|
log "Forking $src_name -> $dst_name"
|
||||||
|
|
||||||
|
# Check if already exists
|
||||||
|
if gh repo view "$LUXFI_ORG/$dst_name" &>/dev/null; then
|
||||||
|
warn "Repo $LUXFI_ORG/$dst_name already exists, skipping fork"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if source exists locally
|
||||||
|
if [[ ! -d "$ZAMA_DIR/$src_name" ]]; then
|
||||||
|
error "Source $ZAMA_DIR/$src_name not found"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Create new repo
|
||||||
|
gh repo create "$LUXFI_ORG/$dst_name" --public --description "Lux FHE: $(echo $dst_name | sed 's/-/ /g')" || {
|
||||||
|
warn "Could not create repo, may already exist"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Clone, strip refs, push
|
||||||
|
local tmp_dir="/tmp/migrate-$$-$dst_name"
|
||||||
|
cp -r "$ZAMA_DIR/$src_name" "$tmp_dir"
|
||||||
|
cd "$tmp_dir"
|
||||||
|
|
||||||
|
# Remove old git history
|
||||||
|
rm -rf .git
|
||||||
|
git init
|
||||||
|
|
||||||
|
# Strip Zama references
|
||||||
|
strip_zama_refs "$tmp_dir"
|
||||||
|
|
||||||
|
# Commit and push
|
||||||
|
git add -A
|
||||||
|
git commit -m "Initial import from Zama (Apache 2.0), migrated to Lux FHE stack"
|
||||||
|
git branch -M main
|
||||||
|
git remote add origin "git@github.com:$LUXFI_ORG/$dst_name.git"
|
||||||
|
git push -u origin main --force || warn "Push failed for $dst_name"
|
||||||
|
|
||||||
|
cd -
|
||||||
|
rm -rf "$tmp_dir"
|
||||||
|
|
||||||
|
log "Successfully migrated $src_name -> $dst_name"
|
||||||
|
}
|
||||||
|
|
||||||
|
strip_zama_refs() {
|
||||||
|
local dir="$1"
|
||||||
|
|
||||||
|
log "Stripping Zama references from $dir"
|
||||||
|
|
||||||
|
# Find all text files
|
||||||
|
find "$dir" -type f \( -name "*.py" -o -name "*.rs" -o -name "*.go" -o -name "*.ts" -o -name "*.js" \
|
||||||
|
-o -name "*.sol" -o -name "*.md" -o -name "*.json" -o -name "*.yaml" -o -name "*.yml" \
|
||||||
|
-o -name "*.toml" -o -name "Makefile" -o -name "*.sh" -o -name "*.txt" \
|
||||||
|
-o -name "Cargo.toml" -o -name "package.json" -o -name "pyproject.toml" \) 2>/dev/null | while read -r file; do
|
||||||
|
|
||||||
|
# Apply replacements
|
||||||
|
for pattern in "${!REPLACEMENTS[@]}"; do
|
||||||
|
replacement="${REPLACEMENTS[$pattern]}"
|
||||||
|
if grep -q "$pattern" "$file" 2>/dev/null; then
|
||||||
|
sed -i '' "s|$pattern|$replacement|g" "$file" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
done
|
||||||
|
|
||||||
|
# Update README files
|
||||||
|
find "$dir" -name "README.md" -type f 2>/dev/null | while read -r readme; do
|
||||||
|
# Add Lux header
|
||||||
|
if ! grep -q "Lux FHE" "$readme" 2>/dev/null; then
|
||||||
|
sed -i '' '1s/^/# Lux FHE\n\nPart of the Lux FHE stack. See https:\/\/github.com\/luxfi\/fhe\n\n/' "$readme" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
update_imports() {
|
||||||
|
local dir="$1"
|
||||||
|
|
||||||
|
log "Updating imports in $dir"
|
||||||
|
|
||||||
|
# Python imports
|
||||||
|
find "$dir" -name "*.py" -type f 2>/dev/null | xargs -I {} sed -i '' \
|
||||||
|
-e 's/from concrete\./from luxfhe./g' \
|
||||||
|
-e 's/import concrete/import luxfhe/g' \
|
||||||
|
-e 's/concrete-ml/fhe-ml/g' \
|
||||||
|
{} 2>/dev/null || true
|
||||||
|
|
||||||
|
# Rust imports
|
||||||
|
find "$dir" -name "*.rs" -o -name "Cargo.toml" 2>/dev/null | xargs -I {} sed -i '' \
|
||||||
|
-e 's/tfhe = /fhe = { git = "https:\/\/github.com\/luxfi\/fhe" } # /g' \
|
||||||
|
-e 's/concrete_/luxfhe_/g' \
|
||||||
|
{} 2>/dev/null || true
|
||||||
|
|
||||||
|
# TypeScript/JavaScript
|
||||||
|
find "$dir" -name "*.ts" -o -name "*.js" -o -name "package.json" 2>/dev/null | xargs -I {} sed -i '' \
|
||||||
|
-e 's/@zama-ai\/fhevm/@luxfi\/fhe/g' \
|
||||||
|
-e 's/fhevm/luxfhe/g' \
|
||||||
|
{} 2>/dev/null || true
|
||||||
|
|
||||||
|
# Go imports
|
||||||
|
find "$dir" -name "*.go" -o -name "go.mod" 2>/dev/null | xargs -I {} sed -i '' \
|
||||||
|
-e 's|github.com/zama-ai/|github.com/luxfi/|g' \
|
||||||
|
{} 2>/dev/null || true
|
||||||
|
}
|
||||||
|
|
||||||
|
# Main migration
|
||||||
|
main() {
|
||||||
|
log "Starting Zama -> Lux FHE migration"
|
||||||
|
log "Source: $ZAMA_DIR"
|
||||||
|
log "Org: $LUXFI_ORG"
|
||||||
|
echo
|
||||||
|
|
||||||
|
# Check prerequisites
|
||||||
|
if ! command -v gh &>/dev/null; then
|
||||||
|
error "GitHub CLI (gh) not found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! gh auth status &>/dev/null; then
|
||||||
|
error "Not logged into GitHub CLI"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Migrate each repo
|
||||||
|
for src_name in "${!MIGRATE_REPOS[@]}"; do
|
||||||
|
dst_name="${MIGRATE_REPOS[$src_name]}"
|
||||||
|
|
||||||
|
echo "---"
|
||||||
|
fork_repo "$src_name" "$dst_name" || warn "Failed to migrate $src_name"
|
||||||
|
done
|
||||||
|
|
||||||
|
log "Migration complete!"
|
||||||
|
echo
|
||||||
|
log "Next steps:"
|
||||||
|
echo " 1. Review each repo and fix any remaining Zama references"
|
||||||
|
echo " 2. Update CI/CD pipelines"
|
||||||
|
echo " 3. Update documentation"
|
||||||
|
echo " 4. Test integrations with luxfi/fhe"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Run with specific repo or all
|
||||||
|
if [[ $# -eq 1 ]]; then
|
||||||
|
src="$1"
|
||||||
|
if [[ -v "MIGRATE_REPOS[$src]" ]]; then
|
||||||
|
fork_repo "$src" "${MIGRATE_REPOS[$src]}"
|
||||||
|
else
|
||||||
|
error "Unknown repo: $src"
|
||||||
|
echo "Available repos:"
|
||||||
|
for r in "${!MIGRATE_REPOS[@]}"; do
|
||||||
|
echo " $r -> ${MIGRATE_REPOS[$r]}"
|
||||||
|
done
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
main
|
||||||
|
fi
|
||||||
@@ -4,10 +4,10 @@ The `fhevm` Command-Line Interface (CLI) tool provides a simple and efficient wa
|
|||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
Ensure you have [Node.js](https://nodejs.org/) installed on your system before proceeding. Then, globally install the `@zama-fhe/relayer-sdk` package to enable the CLI tool:
|
Ensure you have [Node.js](https://nodejs.org/) installed on your system before proceeding. Then, globally install the `@luxfhe/relayer-sdk` package to enable the CLI tool:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
npm install -g @zama-fhe/relayer-sdk
|
npm install -g @luxfhe/relayer-sdk
|
||||||
```
|
```
|
||||||
|
|
||||||
Once installed, you can access the CLI using the `relayer` command. Verify the installation and explore available commands using:
|
Once installed, you can access the CLI using the `relayer` command. Verify the installation and explore available commands using:
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ Public decryption can be done using the Relayer HTTP endpoint.
|
|||||||
Calling the public decryption endpoint of the Relayer can be done easily using the following code snippet.
|
Calling the public decryption endpoint of the Relayer can be done easily using the following code snippet.
|
||||||
|
|
||||||
{% hint style="info" %}
|
{% hint style="info" %}
|
||||||
The total bit length of all ciphertexts being decrypted in a single request must not exceed 2048 bits. Each encrypted type has a specific bit length, for instance `euint8` uses 8 bits and `euint16` uses 16 bits. For the full list of encrypted types and their corresponding bit lengths, refer to the [encrypted types documentation](https://docs.zama.org/protocol/solidity-guides/smart-contract/types#list-of-encrypted-types).
|
The total bit length of all ciphertexts being decrypted in a single request must not exceed 2048 bits. Each encrypted type has a specific bit length, for instance `euint8` uses 8 bits and `euint16` uses 16 bits. For the full list of encrypted types and their corresponding bit lengths, refer to the [encrypted types documentation](https://docs.lux.network/protocol/solidity-guides/smart-contract/types#list-of-encrypted-types).
|
||||||
{% endhint %}
|
{% endhint %}
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
|
|||||||
@@ -48,16 +48,16 @@ For more details on the topic please refer to [the ACL documentation](https://do
|
|||||||
|
|
||||||
## Step 2: decrypt the ciphertext
|
## Step 2: decrypt the ciphertext
|
||||||
|
|
||||||
Using those ciphertext handles, user decryption is performed client-side using the `@zama-fhe/relayer-sdk` library.
|
Using those ciphertext handles, user decryption is performed client-side using the `@luxfhe/relayer-sdk` library.
|
||||||
The `userDecrypt` function takes a **list of handles**, allowing you to decrypt multiple ciphertexts in a single request. In this example, provide just one handle.
|
The `userDecrypt` function takes a **list of handles**, allowing you to decrypt multiple ciphertexts in a single request. In this example, provide just one handle.
|
||||||
The user needs to have created an instance object prior to that (for more context see [the relayer-sdk setup page](./initialization.md)).
|
The user needs to have created an instance object prior to that (for more context see [the relayer-sdk setup page](./initialization.md)).
|
||||||
|
|
||||||
{% hint style="info" %}
|
{% hint style="info" %}
|
||||||
The total bit length of all ciphertexts being decrypted in a single request must not exceed 2048 bits. Each encrypted type has a specific bit length, for instance `euint8` uses 8 bits and `euint16` uses 16 bits. For the full list of encrypted types and their corresponding bit lengths, refer to the [encrypted types documentation](https://docs.zama.org/protocol/solidity-guides/smart-contract/types#list-of-encrypted-types).
|
The total bit length of all ciphertexts being decrypted in a single request must not exceed 2048 bits. Each encrypted type has a specific bit length, for instance `euint8` uses 8 bits and `euint16` uses 16 bits. For the full list of encrypted types and their corresponding bit lengths, refer to the [encrypted types documentation](https://docs.lux.network/protocol/solidity-guides/smart-contract/types#list-of-encrypted-types).
|
||||||
{% endhint %}
|
{% endhint %}
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
// instance: [`FhevmInstance`] from `zama-fhe/relayer-sdk`
|
// instance: [`FhevmInstance`] from `luxfhe/relayer-sdk`
|
||||||
// signer: [`Signer`] from ethers (could a [`Wallet`])
|
// signer: [`Signer`] from ethers (could a [`Wallet`])
|
||||||
// ciphertextHandle: [`string`]
|
// ciphertextHandle: [`string`]
|
||||||
// contractAddress: [`string`]
|
// contractAddress: [`string`]
|
||||||
|
|||||||
@@ -54,7 +54,7 @@ resolve: {
|
|||||||
|
|
||||||
**Cause:** The library may not bundle correctly with certain frameworks, leading to errors during the build or runtime process.
|
**Cause:** The library may not bundle correctly with certain frameworks, leading to errors during the build or runtime process.
|
||||||
|
|
||||||
**Possible solutions:** Use the [prebundled version available](./webapp.md) with `@zama-fhe/relayer-sdk/bundle`. Embed the library with a `<script>` tag and initialize it as shown below:
|
**Possible solutions:** Use the [prebundled version available](./webapp.md) with `@luxfhe/relayer-sdk/bundle`. Embed the library with a `<script>` tag and initialize it as shown below:
|
||||||
|
|
||||||
```javascript
|
```javascript
|
||||||
const start = async () => {
|
const start = async () => {
|
||||||
|
|||||||
+13
@@ -481,6 +481,19 @@ func (eval *ShortIntEvaluator) addCiphertexts(ct1, ct2 *rlwe.Ciphertext) *rlwe.C
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// scaleCiphertext scales a ciphertext by a scalar value
|
||||||
|
func (eval *ShortIntEvaluator) scaleCiphertext(ct *rlwe.Ciphertext, scalar float64) *rlwe.Ciphertext {
|
||||||
|
result := ct.CopyNew()
|
||||||
|
// Scale by multiplying each coefficient
|
||||||
|
for i := range result.Value {
|
||||||
|
coeffs := result.Value[i].Coeffs
|
||||||
|
for j := range coeffs[0] {
|
||||||
|
coeffs[0][j] = uint64(float64(coeffs[0][j]) * scalar)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
// Add adds two shortints (with modular wrap)
|
// Add adds two shortints (with modular wrap)
|
||||||
func (eval *ShortIntEvaluator) Add(a, b *ShortInt) (*ShortInt, error) {
|
func (eval *ShortIntEvaluator) Add(a, b *ShortInt) (*ShortInt, error) {
|
||||||
if a.msgSpace != b.msgSpace {
|
if a.msgSpace != b.msgSpace {
|
||||||
|
|||||||
Reference in New Issue
Block a user