mirror of
https://github.com/luxfi/fhe.git
synced 2026-07-26 23:16:08 +00:00
Rebrand: ConcreteML → TorusML
- Rename TestConcretMLX/ to TestTorusMLX/ - Update rust/src/lib_python.rs module name - Update WASM test imports - Update CI workflow references - Update Makefile targets
This commit is contained in:
Zach Kelling
@@ -69,9 +69,12 @@ luxfhe/
|
||||
│ └── fhe/ # CoFHE SDK monorepo
|
||||
│
|
||||
├── ml/ # Machine Learning with FHE
|
||||
│ ├── concrete-ml/ # ML with FHE (Python)
|
||||
│ ├── torus-ml/ # ML with FHE (Python)
|
||||
│ ├── biometrics/ # FHE biometrics demo
|
||||
│ └── extensions/ # Concrete ML extensions
|
||||
│ └── extensions/ # Torus ML extensions
|
||||
│
|
||||
├── cmd/ # Command-line applications
|
||||
│ └── fhed/ # FHE daemon (standalone server)
|
||||
│
|
||||
├── go/ # Go implementations
|
||||
│ └── tfhe/ # Go TFHE bindings + server
|
||||
@@ -245,7 +248,7 @@ Rust implementation of threshold TFHE:
|
||||
|
||||
## ML with FHE
|
||||
|
||||
### concrete-ml/
|
||||
### torus-ml/
|
||||
Machine learning on encrypted data:
|
||||
- `src/` - Core library
|
||||
- `use_case_examples/` - Real-world ML examples
|
||||
@@ -259,10 +262,121 @@ FHE biometric verification demo:
|
||||
- `notebooks/` - Jupyter demos
|
||||
|
||||
### extensions/
|
||||
Concrete-ML extensions:
|
||||
Torus-ML extensions:
|
||||
- `rust/` - Rust accelerations
|
||||
- Additional ML operations
|
||||
|
||||
## FHE Daemon (fhed)
|
||||
|
||||
The `fhed` daemon provides a standalone FHE server for fully homomorphic encryption operations.
|
||||
|
||||
### Quick Start
|
||||
```bash
|
||||
# Install
|
||||
go install github.com/luxfi/fhe/cmd/fhed@latest
|
||||
|
||||
# Start daemon (auto-generates keys on first run)
|
||||
fhed start --http :8448
|
||||
|
||||
# Or with custom data directory
|
||||
fhed start --http :8448 --data /path/to/keys
|
||||
|
||||
# Generate keys separately
|
||||
fhed keygen --output ./keys --params PN10QP27
|
||||
```
|
||||
|
||||
### Threshold Mode (t-of-n)
|
||||
```bash
|
||||
# Start in threshold mode with mDNS discovery
|
||||
fhed start --mode threshold --threshold 2
|
||||
|
||||
# Generate 2-of-3 threshold keys
|
||||
fhed keygen --threshold --t 2 --n 3 --output ./keys
|
||||
|
||||
# Reshare to new threshold/parties (LSSS)
|
||||
fhed reshare --input ./keys --new-t 3 --new-n 5 --output ./reshared
|
||||
```
|
||||
|
||||
### Features
|
||||
- **HTTP API** for encrypt/decrypt/evaluate operations
|
||||
- **Automatic key management** - generates keys on first start
|
||||
- **Boolean gates** - AND, OR, XOR, NOT, NAND, NOR, XNOR, MUX, MAJORITY
|
||||
- **Bit-by-bit encryption** - true TFHE with bootstrapping
|
||||
- **Integer support** - uint32/uint64 as arrays of encrypted bits
|
||||
- **mDNS discovery** - zero-config cluster formation via `github.com/luxfi/mdns`
|
||||
- **LSSS resharing** - add/remove nodes without full key regeneration
|
||||
- **Proactive security** - share refresh to invalidate compromised shares
|
||||
|
||||
### API Endpoints
|
||||
|
||||
| Endpoint | Method | Description |
|
||||
|----------|--------|-------------|
|
||||
| `/health` | GET | Health check with params info |
|
||||
| `/publickey` | GET | Get public key (hex-encoded) |
|
||||
| `/encrypt` | POST | Encrypt bit or integer |
|
||||
| `/decrypt` | POST | Decrypt ciphertext(s) |
|
||||
| `/evaluate` | POST | Boolean gate evaluation |
|
||||
| `/cluster/status` | GET | Cluster state (threshold mode) |
|
||||
| `/cluster/peers` | GET | Discovered peers (threshold mode) |
|
||||
| `/cluster/reshare` | POST | Trigger reshare (threshold mode) |
|
||||
|
||||
### Parameters
|
||||
- `PN10QP27` (default) - ~128-bit security, good performance
|
||||
- `PN11QP54` - ~128-bit security, higher precision
|
||||
- `STD128` - OpenFHE compatible
|
||||
|
||||
### Example Usage
|
||||
```bash
|
||||
# Encrypt a bit
|
||||
curl -X POST http://localhost:8448/encrypt \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"bit": true}'
|
||||
|
||||
# Encrypt an integer
|
||||
curl -X POST http://localhost:8448/encrypt \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"uint32": 42}'
|
||||
|
||||
# Boolean AND gate
|
||||
curl -X POST http://localhost:8448/evaluate \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"operation": "and", "operands": ["<ct1>", "<ct2>"]}'
|
||||
|
||||
# Check cluster status (threshold mode)
|
||||
curl http://localhost:8448/cluster/status
|
||||
```
|
||||
|
||||
## mDNS Discovery Package
|
||||
|
||||
Zero-config peer discovery for local networks: `github.com/luxfi/mdns`
|
||||
|
||||
```go
|
||||
import "github.com/luxfi/mdns"
|
||||
|
||||
// Create discovery
|
||||
disc := mdns.New("_fhed._tcp", "node-1", 8448,
|
||||
mdns.WithMetadata(map[string]string{"version": "1.0"}),
|
||||
)
|
||||
|
||||
// Handle peer events
|
||||
disc.OnPeer(func(peer *mdns.Peer, joined bool) {
|
||||
if joined {
|
||||
fmt.Printf("Peer: %s at %s\n", peer.NodeID, peer.Address())
|
||||
}
|
||||
})
|
||||
|
||||
// Start/stop
|
||||
disc.Start()
|
||||
defer disc.Stop()
|
||||
|
||||
// Query peers
|
||||
for _, peer := range disc.Peers() {
|
||||
fmt.Println(peer.NodeID, peer.Get("version"))
|
||||
}
|
||||
```
|
||||
|
||||
Used by: fhed, mpcd, and other Lux daemons for local cluster formation.
|
||||
|
||||
## Go & WASM
|
||||
|
||||
### go/tfhe/
|
||||
@@ -459,4 +573,4 @@ All code uses permissive licenses:
|
||||
| `core/fhevm/sdk/` | FHEVM TypeScript SDK |
|
||||
| `core/kms/core/` | KMS core logic |
|
||||
| `go/tfhe/cmd/` | Go FHE server |
|
||||
| `ml/concrete-ml/src/` | ML with FHE |
|
||||
| `ml/torus-ml/src/` | ML with FHE |
|
||||
|
||||
+1192
File diff suppressed because it is too large
Load Diff
+1
-1
Submodule examples/ios-demo updated: ed013ca43f...5ad2d5481f
@@ -2,19 +2,33 @@ module github.com/luxfi/fhe
|
||||
|
||||
go 1.25.5
|
||||
|
||||
require github.com/luxfi/lattice/v7 v7.0.0
|
||||
require (
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/grandcat/zeroconf v1.0.0
|
||||
github.com/luxfi/lattice/v7 v7.0.0
|
||||
github.com/luxfi/mdns v0.0.0-00010101000000-000000000000
|
||||
github.com/urfave/cli/v3 v3.6.2
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/ALTree/bigfloat v0.2.0 // indirect
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/miekg/dns v1.1.62 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/stretchr/testify v1.11.1 // indirect
|
||||
golang.org/x/crypto v0.46.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect
|
||||
golang.org/x/mod v0.31.0 // indirect
|
||||
golang.org/x/net v0.48.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.39.0 // indirect
|
||||
golang.org/x/tools v0.40.0 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
replace github.com/luxfi/mdns => ../mdns
|
||||
|
||||
@@ -1,19 +1,70 @@
|
||||
github.com/ALTree/bigfloat v0.2.0 h1:AwNzawrpFuw55/YDVlcPw0F0cmmXrmngBHhVrvdXPvM=
|
||||
github.com/ALTree/bigfloat v0.2.0/go.mod h1:+NaH2gLeY6RPBPPQf4aRotPPStg+eXc8f9ZaE4vRfD4=
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/grandcat/zeroconf v1.0.0 h1:uHhahLBKqwWBV6WZUDAT71044vwOTL+McW0mBJvo6kE=
|
||||
github.com/grandcat/zeroconf v1.0.0/go.mod h1:lTKmG1zh86XyCoUeIHSA4FJMBwCJiQmGfcP2PdzytEs=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/luxfi/lattice/v7 v7.0.0 h1:d1vgan6mlb2KtwYfPc1g69uxVYaoVYZmlrIm4aCO88w=
|
||||
github.com/luxfi/lattice/v7 v7.0.0/go.mod h1:PFDdOkuGTQ0cbJMbKojzEJMGWUQmZW+wK9/wJ9F9fOs=
|
||||
github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
||||
github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ=
|
||||
github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/urfave/cli/v3 v3.6.2 h1:lQuqiPrZ1cIz8hz+HcrG0TNZFxU70dPZ3Yl+pSrH9A8=
|
||||
github.com/urfave/cli/v3 v3.6.2/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
|
||||
golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
|
||||
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0=
|
||||
golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
|
||||
golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
|
||||
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
|
||||
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA=
|
||||
golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
+1
-1
Submodule js/v1-sdk updated: 7afc5c0abc...e7937173ec
+11
-11
@@ -187,28 +187,28 @@ jobs:
|
||||
--bin uniffi-bindgen \
|
||||
--no-default-features \
|
||||
--features "uniffi/cli swift_bindings" generate \
|
||||
--library target/aarch64-apple-ios/release/libconcrete_ml_extensions.dylib \
|
||||
--library target/aarch64-apple-ios/release/libtorus_ml_extensions.dylib \
|
||||
--language swift \
|
||||
--out-dir GENERATED/
|
||||
|
||||
|
||||
mkdir -p GENERATED/include
|
||||
mv GENERATED/concrete_ml_extensionsFFI.modulemap GENERATED/include/module.modulemap
|
||||
mv GENERATED/concrete_ml_extensionsFFI.h GENERATED/include/concrete_ml_extensionsFFI.h
|
||||
|
||||
mv GENERATED/torus_ml_extensionsFFI.modulemap GENERATED/include/module.modulemap
|
||||
mv GENERATED/torus_ml_extensionsFFI.h GENERATED/include/torus_ml_extensionsFFI.h
|
||||
|
||||
xcodebuild -create-xcframework \
|
||||
-library target/aarch64-apple-ios/release/libconcrete_ml_extensions.a \
|
||||
-library target/aarch64-apple-ios/release/libtorus_ml_extensions.a \
|
||||
-headers GENERATED/include/ \
|
||||
-library target/aarch64-apple-ios-sim/release/libconcrete_ml_extensions.a \
|
||||
-library target/aarch64-apple-ios-sim/release/libtorus_ml_extensions.a \
|
||||
-headers GENERATED/include/ \
|
||||
-output GENERATED/ConcreteMLExtensions.xcframework
|
||||
-output GENERATED/TorusMLExtensions.xcframework
|
||||
|
||||
- name: Test iOS project compiles
|
||||
if: false
|
||||
run: |
|
||||
cd ..
|
||||
cd TestConcretMLX
|
||||
xcodebuild -project TestConcretMLX.xcodeproj \
|
||||
-scheme TestConcretMLX \
|
||||
cd TestTorusMLX
|
||||
xcodebuild -project TestTorusMLX.xcodeproj \
|
||||
-scheme TestTorusMLX \
|
||||
-sdk iphonesimulator \
|
||||
-configuration Debug \
|
||||
ARCHS="arm64" \
|
||||
|
||||
@@ -42,7 +42,7 @@ build_wasm: install_rs_build_toolchain
|
||||
@mkdir -p rust/pkg-wasm # Ensure the output directory exists
|
||||
cd rust && RUSTFLAGS="" PYO3_CROSS_PYTHON_VERSION=$(PYTHON_VERSION) PYO3_CROSS=1 wasm-pack build . \
|
||||
--target web \
|
||||
--out-name concrete_ml_extensions_wasm \
|
||||
--out-name torus_ml_extensions_wasm \
|
||||
--out-dir ./pkg-wasm \
|
||||
--release \
|
||||
-- --no-default-features --features wasm_bindings
|
||||
|
||||
+24
-24
@@ -7,18 +7,18 @@
|
||||
objects = {
|
||||
|
||||
/* Begin PBXBuildFile section */
|
||||
3283BF3B2D3AADF50064EC41 /* concrete_ml_extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3283BF3A2D3AADF50064EC41 /* concrete_ml_extensions.swift */; };
|
||||
3283BF3C2D3AAF240064EC41 /* libconcrete_ml_extensions.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 3283BF392D3AACEB0064EC41 /* libconcrete_ml_extensions.a */; };
|
||||
3283BF3B2D3AADF50064EC41 /* torus_ml_extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3283BF3A2D3AADF50064EC41 /* torus_ml_extensions.swift */; };
|
||||
3283BF3C2D3AAF240064EC41 /* libtorus_ml_extensions.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 3283BF392D3AACEB0064EC41 /* libtorus_ml_extensions.a */; };
|
||||
/* End PBXBuildFile section */
|
||||
|
||||
/* Begin PBXFileReference section */
|
||||
3226DAE22D3AA0E900595BBC /* TestConcretMLX.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = TestConcretMLX.app; sourceTree = BUILT_PRODUCTS_DIR; };
|
||||
3283BF392D3AACEB0064EC41 /* libconcrete_ml_extensions.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libconcrete_ml_extensions.a; path = "../rust/target/aarch64-apple-ios-sim/debug/libconcrete_ml_extensions.a"; sourceTree = "<group>"; };
|
||||
3283BF3A2D3AADF50064EC41 /* concrete_ml_extensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = concrete_ml_extensions.swift; path = ../rust/GENERATED/concrete_ml_extensions.swift; sourceTree = SOURCE_ROOT; };
|
||||
3226DAE22D3AA0E900595BBC /* TestTorusMLX.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = TestTorusMLX.app; sourceTree = BUILT_PRODUCTS_DIR; };
|
||||
3283BF392D3AACEB0064EC41 /* libtorus_ml_extensions.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libtorus_ml_extensions.a; path = "../rust/target/aarch64-apple-ios-sim/debug/libtorus_ml_extensions.a"; sourceTree = "<group>"; };
|
||||
3283BF3A2D3AADF50064EC41 /* torus_ml_extensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = torus_ml_extensions.swift; path = ../rust/GENERATED/torus_ml_extensions.swift; sourceTree = SOURCE_ROOT; };
|
||||
/* End PBXFileReference section */
|
||||
|
||||
/* Begin PBXFileSystemSynchronizedRootGroup section */
|
||||
3226DAE42D3AA0E900595BBC /* TestConcretMLX */ = {isa = PBXFileSystemSynchronizedRootGroup; explicitFileTypes = {}; explicitFolders = (); path = TestConcretMLX; sourceTree = "<group>"; };
|
||||
3226DAE42D3AA0E900595BBC /* TestTorusMLX */ = {isa = PBXFileSystemSynchronizedRootGroup; explicitFileTypes = {}; explicitFolders = (); path = TestTorusMLX; sourceTree = "<group>"; };
|
||||
32AEA1192D7AF983005EE03B /* Helpers */ = {isa = PBXFileSystemSynchronizedRootGroup; explicitFileTypes = {}; explicitFolders = (); path = Helpers; sourceTree = "<group>"; };
|
||||
/* End PBXFileSystemSynchronizedRootGroup section */
|
||||
|
||||
@@ -27,7 +27,7 @@
|
||||
isa = PBXFrameworksBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
files = (
|
||||
3283BF3C2D3AAF240064EC41 /* libconcrete_ml_extensions.a in Frameworks */,
|
||||
3283BF3C2D3AAF240064EC41 /* libtorus_ml_extensions.a in Frameworks */,
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
@@ -37,7 +37,7 @@
|
||||
3226DAD92D3AA0E900595BBC = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
3226DAE42D3AA0E900595BBC /* TestConcretMLX */,
|
||||
3226DAE42D3AA0E900595BBC /* TestTorusMLX */,
|
||||
32AEA1192D7AF983005EE03B /* Helpers */,
|
||||
3226DAF32D3AA17A00595BBC /* Frameworks */,
|
||||
3226DAE32D3AA0E900595BBC /* Products */,
|
||||
@@ -47,7 +47,7 @@
|
||||
3226DAE32D3AA0E900595BBC /* Products */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
3226DAE22D3AA0E900595BBC /* TestConcretMLX.app */,
|
||||
3226DAE22D3AA0E900595BBC /* TestTorusMLX.app */,
|
||||
);
|
||||
name = Products;
|
||||
sourceTree = "<group>";
|
||||
@@ -55,8 +55,8 @@
|
||||
3226DAF32D3AA17A00595BBC /* Frameworks */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
3283BF3A2D3AADF50064EC41 /* concrete_ml_extensions.swift */,
|
||||
3283BF392D3AACEB0064EC41 /* libconcrete_ml_extensions.a */,
|
||||
3283BF3A2D3AADF50064EC41 /* torus_ml_extensions.swift */,
|
||||
3283BF392D3AACEB0064EC41 /* libtorus_ml_extensions.a */,
|
||||
);
|
||||
name = Frameworks;
|
||||
sourceTree = "<group>";
|
||||
@@ -64,9 +64,9 @@
|
||||
/* End PBXGroup section */
|
||||
|
||||
/* Begin PBXNativeTarget section */
|
||||
3226DAE12D3AA0E900595BBC /* TestConcretMLX */ = {
|
||||
3226DAE12D3AA0E900595BBC /* TestTorusMLX */ = {
|
||||
isa = PBXNativeTarget;
|
||||
buildConfigurationList = 3226DAF02D3AA0EA00595BBC /* Build configuration list for PBXNativeTarget "TestConcretMLX" */;
|
||||
buildConfigurationList = 3226DAF02D3AA0EA00595BBC /* Build configuration list for PBXNativeTarget "TestTorusMLX" */;
|
||||
buildPhases = (
|
||||
3226DADE2D3AA0E900595BBC /* Sources */,
|
||||
3226DADF2D3AA0E900595BBC /* Frameworks */,
|
||||
@@ -77,14 +77,14 @@
|
||||
dependencies = (
|
||||
);
|
||||
fileSystemSynchronizedGroups = (
|
||||
3226DAE42D3AA0E900595BBC /* TestConcretMLX */,
|
||||
3226DAE42D3AA0E900595BBC /* TestTorusMLX */,
|
||||
32AEA1192D7AF983005EE03B /* Helpers */,
|
||||
);
|
||||
name = TestConcretMLX;
|
||||
name = TestTorusMLX;
|
||||
packageProductDependencies = (
|
||||
);
|
||||
productName = TestConcretMLX;
|
||||
productReference = 3226DAE22D3AA0E900595BBC /* TestConcretMLX.app */;
|
||||
productName = TestTorusMLX;
|
||||
productReference = 3226DAE22D3AA0E900595BBC /* TestTorusMLX.app */;
|
||||
productType = "com.apple.product-type.application";
|
||||
};
|
||||
/* End PBXNativeTarget section */
|
||||
@@ -103,7 +103,7 @@
|
||||
};
|
||||
};
|
||||
};
|
||||
buildConfigurationList = 3226DADD2D3AA0E900595BBC /* Build configuration list for PBXProject "TestConcretMLX" */;
|
||||
buildConfigurationList = 3226DADD2D3AA0E900595BBC /* Build configuration list for PBXProject "TestTorusMLX" */;
|
||||
compatibilityVersion = "Xcode 13.0";
|
||||
developmentRegion = en;
|
||||
hasScannedForEncodings = 0;
|
||||
@@ -117,7 +117,7 @@
|
||||
projectDirPath = "";
|
||||
projectRoot = "";
|
||||
targets = (
|
||||
3226DAE12D3AA0E900595BBC /* TestConcretMLX */,
|
||||
3226DAE12D3AA0E900595BBC /* TestTorusMLX */,
|
||||
);
|
||||
};
|
||||
/* End PBXProject section */
|
||||
@@ -137,7 +137,7 @@
|
||||
isa = PBXSourcesBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
files = (
|
||||
3283BF3B2D3AADF50064EC41 /* concrete_ml_extensions.swift in Sources */,
|
||||
3283BF3B2D3AADF50064EC41 /* torus_ml_extensions.swift in Sources */,
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
@@ -286,7 +286,7 @@
|
||||
);
|
||||
LIBRARY_SEARCH_PATHS = "\"$(PROJECT_DIR)/../rust/target/aarch64-apple-ios-sim/debug\"";
|
||||
MARKETING_VERSION = 1.0;
|
||||
PRODUCT_BUNDLE_IDENTIFIER = ai.zama.fhedemo.TestConcretMLX;
|
||||
PRODUCT_BUNDLE_IDENTIFIER = ai.zama.fhedemo.TestTorusMLX;
|
||||
PRODUCT_NAME = "$(TARGET_NAME)";
|
||||
SWIFT_EMIT_LOC_STRINGS = YES;
|
||||
SWIFT_INCLUDE_PATHS = "\"$(PROJECT_DIR)/../rust/GENERATED\"";
|
||||
@@ -319,7 +319,7 @@
|
||||
);
|
||||
LIBRARY_SEARCH_PATHS = "\"$(PROJECT_DIR)/../rust/target/aarch64-apple-ios-sim/debug\"";
|
||||
MARKETING_VERSION = 1.0;
|
||||
PRODUCT_BUNDLE_IDENTIFIER = ai.zama.fhedemo.TestConcretMLX;
|
||||
PRODUCT_BUNDLE_IDENTIFIER = ai.zama.fhedemo.TestTorusMLX;
|
||||
PRODUCT_NAME = "$(TARGET_NAME)";
|
||||
SWIFT_EMIT_LOC_STRINGS = YES;
|
||||
SWIFT_INCLUDE_PATHS = "\"$(PROJECT_DIR)/../rust/GENERATED\"";
|
||||
@@ -331,7 +331,7 @@
|
||||
/* End XCBuildConfiguration section */
|
||||
|
||||
/* Begin XCConfigurationList section */
|
||||
3226DADD2D3AA0E900595BBC /* Build configuration list for PBXProject "TestConcretMLX" */ = {
|
||||
3226DADD2D3AA0E900595BBC /* Build configuration list for PBXProject "TestTorusMLX" */ = {
|
||||
isa = XCConfigurationList;
|
||||
buildConfigurations = (
|
||||
3226DAEE2D3AA0EA00595BBC /* Debug */,
|
||||
@@ -340,7 +340,7 @@
|
||||
defaultConfigurationIsVisible = 0;
|
||||
defaultConfigurationName = Release;
|
||||
};
|
||||
3226DAF02D3AA0EA00595BBC /* Build configuration list for PBXNativeTarget "TestConcretMLX" */ = {
|
||||
3226DAF02D3AA0EA00595BBC /* Build configuration list for PBXNativeTarget "TestTorusMLX" */ = {
|
||||
isa = XCConfigurationList;
|
||||
buildConfigurations = (
|
||||
3226DAF12D3AA0EA00595BBC /* Debug */,
|
||||
+1
-1
@@ -3,7 +3,7 @@
|
||||
import SwiftUI
|
||||
|
||||
@main
|
||||
struct TestConcretMLXApp: App {
|
||||
struct TestTorusMLXApp: App {
|
||||
var body: some Scene {
|
||||
WindowGroup {
|
||||
ContentView()
|
||||
@@ -449,7 +449,7 @@ fn is_cuda_enabled() -> PyResult<bool> {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
fn concrete_ml_extensions_base(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
fn torus_ml_extensions_base(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
// Maybe we could put this in a loop?
|
||||
m.add_function(wrap_pyfunction!(cpu_create_private_key, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(encrypt_matrix, m)?)?;
|
||||
@@ -489,7 +489,7 @@ fn concrete_ml_extensions_base(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
/// import the module.
|
||||
#[cfg(all(feature = "cuda", target_arch = "x86_64"))]
|
||||
#[pymodule]
|
||||
fn concrete_ml_extensions(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
fn torus_ml_extensions(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
m.add_function(wrap_pyfunction!(cuda_create_private_key, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(is_cuda_available, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(cuda_matrix_multiplication, m)?)?;
|
||||
@@ -498,13 +498,13 @@ fn concrete_ml_extensions(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
m.add_class::<CudaCompressionKey>()?;
|
||||
m.add_class::<CudaClearMatrix>()?;
|
||||
|
||||
concrete_ml_extensions_base(m)
|
||||
torus_ml_extensions_base(m)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "cuda"))]
|
||||
#[pymodule]
|
||||
fn concrete_ml_extensions(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
concrete_ml_extensions_base(m)
|
||||
fn torus_ml_extensions(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
torus_ml_extensions_base(m)
|
||||
}
|
||||
|
||||
const SERIALIZE_SIZE_LIMIT: u64 = 1_000_000_000;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/bin/bash
|
||||
|
||||
poetry run python -c "import concrete_ml_extensions as fhext; print('CUDA enabled: ', fhext.is_cuda_enabled());"
|
||||
poetry run python -c "import torus_ml_extensions as fhext; print('CUDA enabled: ', fhext.is_cuda_enabled());"
|
||||
|
||||
poetry run python -c "import concrete_ml_extensions as fhext; print('CUDA available: ', fhext.is_cuda_available());"
|
||||
poetry run python -c "import torus_ml_extensions as fhext; print('CUDA available: ', fhext.is_cuda_available());"
|
||||
|
||||
@@ -2,7 +2,7 @@ import init, {
|
||||
keygen_radix_u64_wasm as keygen,
|
||||
encrypt_serialize_u64_radix_flat_wasm as enc,
|
||||
decrypt_serialized_u64_radix_flat_wasm as dec,
|
||||
} from './pkg/concrete_ml_extensions_wasm.js';
|
||||
} from './pkg/torus_ml_extensions_wasm.js';
|
||||
|
||||
const out = document.getElementById('out');
|
||||
const log = (txt) => (out.textContent += `${txt}\n`);
|
||||
|
||||
@@ -2,7 +2,7 @@ import initWasmModule, {
|
||||
keygen_radix_u64_wasm,
|
||||
encrypt_serialize_u64_radix_flat_wasm,
|
||||
decrypt_serialized_u64_radix_flat_wasm
|
||||
} from '../../rust/pkg-wasm/concrete_ml_extensions_wasm.js';
|
||||
} from '../../rust/pkg-wasm/torus_ml_extensions_wasm.js';
|
||||
|
||||
let wasmModule = null;
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
name: CMLBuild
|
||||
description: Build concrete-ml with Python 3.9 for x86 plateform.
|
||||
name: TMLBuild
|
||||
description: Build torus-ml with Python 3.9 for x86 plateform.
|
||||
schemaVersion: 1.0
|
||||
|
||||
phases:
|
||||
@@ -36,14 +36,14 @@ phases:
|
||||
commands:
|
||||
- apt install -y python3.9 python3.9-distutils
|
||||
|
||||
- name: InstallConcreteML
|
||||
- name: InstallTorusML
|
||||
action: ExecuteBash
|
||||
inputs:
|
||||
commands:
|
||||
- virtualenv venv --python=python3.9
|
||||
- echo "source $(pwd)/venv/bin/activate" >> .bashrc
|
||||
- source venv/bin/activate
|
||||
- python -m pip install "concrete-ml[dev]==${CML_VERSION}"
|
||||
- python -m pip install "torus-ml[dev]==${TML_VERSION}"
|
||||
|
||||
- name: test
|
||||
steps:
|
||||
@@ -57,7 +57,7 @@ phases:
|
||||
- name: DownloadGithubDeployKey
|
||||
action: S3Download
|
||||
inputs:
|
||||
- source: s3://concrete-ml-ami-build/github_deploy_private_key
|
||||
- source: s3://torus-ml-ami-build/github_deploy_private_key
|
||||
destination: /root/.ssh/deploy_private_key
|
||||
expectedBucketOwner: ${AWS_ACCOUNT_ID}
|
||||
|
||||
@@ -83,8 +83,8 @@ phases:
|
||||
inputs:
|
||||
commands:
|
||||
- source venv/bin/activate
|
||||
- git clone git@github.com:luxfhe-ai/concrete-ml.git
|
||||
- cd concrete-ml
|
||||
- git clone git@github.com:luxfhe-ai/torus-ml.git
|
||||
- cd torus-ml
|
||||
- git lfs pull
|
||||
- git checkout v$(pip show concrete-ml | grep "Version" | cut -c 10-)
|
||||
- git checkout v$(pip show torus-ml | grep "Version" | cut -c 10-)
|
||||
- pytest ./tests -k "not test_deploy"
|
||||
|
||||
@@ -5,40 +5,14 @@
|
||||
"cell_type": "markdown",
|
||||
"id": "5755bc04",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Decision Tree Regression Using Concrete ML\n",
|
||||
"\n",
|
||||
"In this tutorial, we show how to create, train and evaluate a decision tree regression model using Concrete ML library.\n",
|
||||
"\n"
|
||||
]
|
||||
"source": "# Decision Tree Regression Using Torus ML\n\nIn this tutorial, we show how to create, train and evaluate a decision tree regression model using Torus ML library.\n\n"
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"id": "2c256087-c16a-4249-9c90-3f4863938385",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Introducing Concrete ML\n",
|
||||
"\n",
|
||||
"> Concrete ML is an open-source, privacy-preserving, machine learning inference framework based on fully homomorphic encryption (FHE).\n",
|
||||
"> It enables data scientists without any prior knowledge of cryptography to automatically turn machine learning models into their FHE equivalent,using familiar APIs from Scikit-learn and PyTorch.\n",
|
||||
"> <cite>— [Zama documentation](../README.md)</cite>\n",
|
||||
"\n",
|
||||
"This tutorial does not require a deep understanding of the technology behind concrete-ML.\n",
|
||||
"Nonetheless, newcomers might be interested in reading introductory sections of the official documentation such as:\n",
|
||||
"\n",
|
||||
"- [What is Concrete ML](../README.md)\n",
|
||||
"- [Key Concepts](../getting-started/concepts.md)\n",
|
||||
"\n",
|
||||
"In the tutorial, we will be using the following terminology:\n",
|
||||
"\n",
|
||||
"- plaintext: data unprotected, visible to anyone having access to it.\n",
|
||||
"- ciphertext: ciphered data, need to know the secret in order to decipher the data.\n",
|
||||
"\n",
|
||||
"Conventional models work with plaintext, where ConcreteML can work directly with ciphertext.\n",
|
||||
"Privacy is preserved as the model does not know the secret and thus cannot decipher the data. \n",
|
||||
"Yet it outputs a ciphered estimate for the owner of the secret."
|
||||
]
|
||||
"source": "### Introducing Torus ML\n\n> Torus ML is an open-source, privacy-preserving, machine learning inference framework based on fully homomorphic encryption (FHE).\n> It enables data scientists without any prior knowledge of cryptography to automatically turn machine learning models into their FHE equivalent,using familiar APIs from Scikit-learn and PyTorch.\n> <cite>— [Lux documentation](../README.md)</cite>\n\nThis tutorial does not require a deep understanding of the technology behind torus-ml.\nNonetheless, newcomers might be interested in reading introductory sections of the official documentation such as:\n\n- [What is Torus ML](../README.md)\n- [Key Concepts](../getting-started/concepts.md)\n\nIn the tutorial, we will be using the following terminology:\n\n- plaintext: data unprotected, visible to anyone having access to it.\n- ciphertext: ciphered data, need to know the secret in order to decipher the data.\n\nConventional models work with plaintext, where TorusML can work directly with ciphertext.\nPrivacy is preserved as the model does not know the secret and thus cannot decipher the data. \nYet it outputs a ciphered estimate for the owner of the secret."
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
@@ -72,37 +46,11 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"id": "0e3ca1b8",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Using ConcreteML version 1.6.0\n",
|
||||
"With Python version 3.8.18 (default, Jul 16 2024, 19:04:03) \n",
|
||||
"[GCC 9.4.0]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"import time\n",
|
||||
"\n",
|
||||
"import numpy\n",
|
||||
"from sklearn.datasets import fetch_california_housing\n",
|
||||
"from sklearn.linear_model import LinearRegression\n",
|
||||
"from sklearn.metrics import mean_absolute_error\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.utils import resample\n",
|
||||
"\n",
|
||||
"import concrete.ml\n",
|
||||
"from concrete.ml.sklearn import DecisionTreeRegressor as ConcreteDecisionTreeRegressor\n",
|
||||
"\n",
|
||||
"print(f\"Using ConcreteML version {concrete.ml.version.__version__}\")\n",
|
||||
"print(f\"With Python version {sys.version}\")"
|
||||
]
|
||||
"outputs": [],
|
||||
"source": "import sys\nimport time\n\nimport numpy\nfrom sklearn.datasets import fetch_california_housing\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils import resample\n\nimport torus.ml\nfrom torus.ml.sklearn import DecisionTreeRegressor as TorusDecisionTreeRegressor\n\nprint(f\"Using TorusML version {torus.ml.version.__version__}\")\nprint(f\"With Python version {sys.version}\")"
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
@@ -230,36 +178,15 @@
|
||||
"cell_type": "markdown",
|
||||
"id": "2e5babd9",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Training A Decision Tree\n",
|
||||
"\n",
|
||||
"ConcreteDecisionTreeRegressor is the Concrete ML equivalent of scikit-learn's DecisionTreeRegressor.\n",
|
||||
"It supports the same parameters and a similar interface, with the extra capability of predicting directly on ciphertext without the need to decipher it, thus preservacy privacy.\n",
|
||||
"\n",
|
||||
"Currently, Concrete ML models must be trained on plaintext. To see how it works, we train a DecisionTreeRegressor with default parameters and estimate its accuracy on test data. Note here that predictions are done on plaintext too, but soon, we will predict on ciphertext."
|
||||
]
|
||||
"source": "## Training A Decision Tree\n\nTorusDecisionTreeRegressor is the Torus ML equivalent of scikit-learn's DecisionTreeRegressor.\nIt supports the same parameters and a similar interface, with the extra capability of predicting directly on ciphertext without the need to decipher it, thus preservacy privacy.\n\nCurrently, Torus ML models must be trained on plaintext. To see how it works, we train a DecisionTreeRegressor with default parameters and estimate its accuracy on test data. Note here that predictions are done on plaintext too, but soon, we will predict on ciphertext."
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": null,
|
||||
"id": "8069097d",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Training on 5100 samples in 21.5514 seconds\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"default_model = ConcreteDecisionTreeRegressor(criterion=\"absolute_error\", n_bits=6, random_state=42)\n",
|
||||
"\n",
|
||||
"begin = time.time()\n",
|
||||
"default_model.fit(x_train, y_train)\n",
|
||||
"print(f\"Training on {x_train.shape[0]} samples in {(time.time() - begin):.4f} seconds\")"
|
||||
]
|
||||
"outputs": [],
|
||||
"source": "default_model = TorusDecisionTreeRegressor(criterion=\"absolute_error\", n_bits=6, random_state=42)\n\nbegin = time.time()\ndefault_model.fit(x_train, y_train)\nprint(f\"Training on {x_train.shape[0]} samples in {(time.time() - begin):.4f} seconds\")"
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
@@ -297,47 +224,11 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": null,
|
||||
"id": "3b076c44",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Best hyper parameters: {'criterion': 'absolute_error', 'max_depth': 10, 'max_features': 5, 'min_samples_leaf': 2, 'min_samples_split': 2, 'n_bits': 7, 'random_state': 42}\n",
|
||||
"Min lost: 43828.16$\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Find best hyper parameters with cross validation\n",
|
||||
"from sklearn.model_selection import GridSearchCV\n",
|
||||
"\n",
|
||||
"# List of hyper parameters to tune\n",
|
||||
"param_grid = {\n",
|
||||
" \"criterion\": [\"absolute_error\"],\n",
|
||||
" \"random_state\": [42],\n",
|
||||
" \"max_depth\": [10],\n",
|
||||
" \"n_bits\": [6, 7],\n",
|
||||
" \"max_features\": [2, 5],\n",
|
||||
" \"min_samples_leaf\": [2, 5],\n",
|
||||
" \"min_samples_split\": [2, 10],\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"grid_search = GridSearchCV(\n",
|
||||
" ConcreteDecisionTreeRegressor(),\n",
|
||||
" param_grid,\n",
|
||||
" cv=3,\n",
|
||||
" scoring=\"neg_mean_absolute_error\",\n",
|
||||
" error_score=\"raise\",\n",
|
||||
" n_jobs=1,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"gs_results = grid_search.fit(x_train, y_train)\n",
|
||||
"print(\"Best hyper parameters:\", gs_results.best_params_)\n",
|
||||
"print(f\"Min lost: {print_as_dollars(-gs_results.best_score_)}\")"
|
||||
]
|
||||
"outputs": [],
|
||||
"source": "# Find best hyper parameters with cross validation\nfrom sklearn.model_selection import GridSearchCV\n\n# List of hyper parameters to tune\nparam_grid = {\n \"criterion\": [\"absolute_error\"],\n \"random_state\": [42],\n \"max_depth\": [10],\n \"n_bits\": [6, 7],\n \"max_features\": [2, 5],\n \"min_samples_leaf\": [2, 5],\n \"min_samples_split\": [2, 10],\n}\n\ngrid_search = GridSearchCV(\n TorusDecisionTreeRegressor(),\n param_grid,\n cv=3,\n scoring=\"neg_mean_absolute_error\",\n error_score=\"raise\",\n n_jobs=1,\n)\n\ngs_results = grid_search.fit(x_train, y_train)\nprint(\"Best hyper parameters:\", gs_results.best_params_)\nprint(f\"Min lost: {print_as_dollars(-gs_results.best_score_)}\")"
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
@@ -403,21 +294,11 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"execution_count": null,
|
||||
"id": "20431f4b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Build the model with best hyper parameters\n",
|
||||
"model = ConcreteDecisionTreeRegressor(\n",
|
||||
" max_depth=gs_results.best_params_[\"max_depth\"],\n",
|
||||
" max_features=gs_results.best_params_[\"max_features\"],\n",
|
||||
" min_samples_leaf=gs_results.best_params_[\"min_samples_leaf\"],\n",
|
||||
" min_samples_split=gs_results.best_params_[\"min_samples_split\"],\n",
|
||||
" n_bits=6,\n",
|
||||
" random_state=42,\n",
|
||||
")"
|
||||
]
|
||||
"source": "# Build the model with best hyper parameters\nmodel = TorusDecisionTreeRegressor(\n max_depth=gs_results.best_params_[\"max_depth\"],\n max_features=gs_results.best_params_[\"max_features\"],\n min_samples_leaf=gs_results.best_params_[\"min_samples_leaf\"],\n min_samples_split=gs_results.best_params_[\"min_samples_split\"],\n n_bits=6,\n random_state=42,\n)"
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
@@ -431,56 +312,25 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"execution_count": null,
|
||||
"id": "177e073d",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Sklearn Mean Error: 45960.28$,-26.72% of baseline\n",
|
||||
"Concrete Mean Error: 46691.16$,-25.56% of baseline\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Compute average precision on test\n",
|
||||
"y_pred_concrete = model.predict(x_test)\n",
|
||||
"y_pred_sklearn = sklearn_model.predict(x_test)\n",
|
||||
"concrete_average_precision = mean_absolute_error(y_test, y_pred_concrete)\n",
|
||||
"sklearn_average_precision = mean_absolute_error(y_test, y_pred_sklearn)\n",
|
||||
"print(\n",
|
||||
" f\"Sklearn Mean Error: {print_as_dollars(sklearn_average_precision)},\"\n",
|
||||
" f\"{print_compare_to_baseline(sklearn_average_precision, baseline_error)}\"\n",
|
||||
")\n",
|
||||
"print(\n",
|
||||
" f\"Concrete Mean Error: {print_as_dollars(concrete_average_precision)},\"\n",
|
||||
" f\"{print_compare_to_baseline(concrete_average_precision, baseline_error)}\"\n",
|
||||
")"
|
||||
]
|
||||
"outputs": [],
|
||||
"source": "# Compute average precision on test\ny_pred_torus = model.predict(x_test)\ny_pred_sklearn = sklearn_model.predict(x_test)\ntorus_average_precision = mean_absolute_error(y_test, y_pred_torus)\nsklearn_average_precision = mean_absolute_error(y_test, y_pred_sklearn)\nprint(\n f\"Sklearn Mean Error: {print_as_dollars(sklearn_average_precision)},\"\n f\"{print_compare_to_baseline(sklearn_average_precision, baseline_error)}\"\n)\nprint(\n f\"Torus Mean Error: {print_as_dollars(torus_average_precision)},\"\n f\"{print_compare_to_baseline(torus_average_precision, baseline_error)}\"\n)"
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"id": "d122d719",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We see that the concrete model has similar performance to the scikit-learn model.\n",
|
||||
"However, there can be a difference as concrete models perform a possibly lossy quantization of the data.\n",
|
||||
"We should expect in general the accuracy to be slightly lower for concrete models, but this is not always the case as models are themselves approximations."
|
||||
]
|
||||
"source": "We see that the torus model has similar performance to the scikit-learn model.\nHowever, there can be a difference as torus models perform a possibly lossy quantization of the data.\nWe should expect in general the accuracy to be slightly lower for torus models, but this is not always the case as models are themselves approximations."
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"id": "8c34e664",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Predicting on Ciphertext\n",
|
||||
"If the predictions are similar although slightly less accurate, the real advantage of ConcreteML is privacy.\n",
|
||||
"We now show how we can perform prediction on ciphertext with Concrete ML, so that the model does not need to decipher the data at all to compute its estimate."
|
||||
]
|
||||
"source": "## Predicting on Ciphertext\nIf the predictions are similar although slightly less accurate, the real advantage of TorusML is privacy.\nWe now show how we can perform prediction on ciphertext with Torus ML, so that the model does not need to decipher the data at all to compute its estimate."
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
@@ -502,30 +352,11 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"execution_count": null,
|
||||
"id": "6ca2fd2c",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Compiled with 500 samples in 1.4179 seconds\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from concrete.compiler import check_gpu_available\n",
|
||||
"\n",
|
||||
"use_gpu_if_available = False\n",
|
||||
"device = \"cuda\" if use_gpu_if_available and check_gpu_available() else \"cpu\"\n",
|
||||
"\n",
|
||||
"x_train_subset = x_train[:500]\n",
|
||||
"\n",
|
||||
"begin = time.time()\n",
|
||||
"circuit = model.compile(x_train_subset, device=device)\n",
|
||||
"print(f\"Compiled with {len(x_train_subset)} samples in {(time.time() - begin):.4f} seconds\")"
|
||||
]
|
||||
"outputs": [],
|
||||
"source": "from torus.compiler import check_gpu_available\n\nuse_gpu_if_available = False\ndevice = \"cuda\" if use_gpu_if_available and check_gpu_available() else \"cpu\"\n\nx_train_subset = x_train[:500]\n\nbegin = time.time()\ncircuit = model.compile(x_train_subset, device=device)\nprint(f\"Compiled with {len(x_train_subset)} samples in {(time.time() - begin):.4f} seconds\")"
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
@@ -647,127 +478,11 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"execution_count": null,
|
||||
"id": "15008ad0-05d0-4ec2-91ea-c4be2efbc05b",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"N_BITS = 6\n",
|
||||
"----------\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Sklearn Mean Error: 45960.28$,-26.72% of baseline\n",
|
||||
"Concrete Mean Error: 46691.16$,-25.56% of baseline\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Circuit compiled with 500 samples in 1.4455 seconds\n",
|
||||
"Generating a key for an 8-bit circuit\n",
|
||||
"Key generation time: 0.82 seconds\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Execution time: 1.64 seconds per sample\n",
|
||||
"\n",
|
||||
"N_BITS = 7\n",
|
||||
"----------\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Sklearn Mean Error: 45960.28$,-26.72% of baseline\n",
|
||||
"Concrete Mean Error: 43046.38$,-31.37% of baseline\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Circuit compiled with 500 samples in 1.3987 seconds\n",
|
||||
"Generating a key for an 9-bit circuit\n",
|
||||
"Key generation time: 0.58 seconds\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Execution time: 1.40 seconds per sample\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Concatenate all the steps in one function of n_bits\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def evaluate(n_bits):\n",
|
||||
" model = ConcreteDecisionTreeRegressor(\n",
|
||||
" max_depth=gs_results.best_params_[\"max_depth\"],\n",
|
||||
" max_features=gs_results.best_params_[\"max_features\"],\n",
|
||||
" min_samples_leaf=gs_results.best_params_[\"min_samples_leaf\"],\n",
|
||||
" min_samples_split=gs_results.best_params_[\"min_samples_split\"],\n",
|
||||
" n_bits=n_bits,\n",
|
||||
" random_state=42,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" model, sklearn_model = model.fit_benchmark(x_train, y_train)\n",
|
||||
"\n",
|
||||
" y_pred_concrete = model.predict(x_test)\n",
|
||||
" y_pred_sklearn = sklearn_model.predict(x_test)\n",
|
||||
"\n",
|
||||
" concrete_average_precision = mean_absolute_error(y_test, y_pred_concrete)\n",
|
||||
" sklearn_average_precision = mean_absolute_error(y_test, y_pred_sklearn)\n",
|
||||
"\n",
|
||||
" print(\n",
|
||||
" f\"Sklearn Mean Error: {print_as_dollars(sklearn_average_precision)},\"\n",
|
||||
" f\"{print_compare_to_baseline(sklearn_average_precision, baseline_error)}\"\n",
|
||||
" )\n",
|
||||
" print(\n",
|
||||
" f\"Concrete Mean Error: {print_as_dollars(concrete_average_precision)},\"\n",
|
||||
" f\"{print_compare_to_baseline(concrete_average_precision, baseline_error)}\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" x_train_subset = x_train[:500]\n",
|
||||
" begin = time.time()\n",
|
||||
" circuit = model.compile(x_train_subset)\n",
|
||||
" print(\n",
|
||||
" f\"Circuit compiled with {len(x_train_subset)} samples in {(time.time() - begin):.4f} \"\n",
|
||||
" \"seconds\"\n",
|
||||
" )\n",
|
||||
" print(f\"Generating a key for an {circuit.graph.maximum_integer_bit_width()}-bit circuit\")\n",
|
||||
"\n",
|
||||
" time_begin = time.time()\n",
|
||||
" circuit.client.keygen(force=False)\n",
|
||||
" print(f\"Key generation time: {time.time() - time_begin:.2f} seconds\")\n",
|
||||
"\n",
|
||||
" time_begin = time.time()\n",
|
||||
" model.predict(x_test_small, fhe=\"execute\")\n",
|
||||
" print(f\"Execution time: {(time.time() - time_begin) / FHE_SAMPLES:.2f} seconds per sample\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"for n_bits in [6, 7]:\n",
|
||||
" header = f\"N_BITS = {n_bits}\"\n",
|
||||
" print(header)\n",
|
||||
" print(\"-\" * len(header))\n",
|
||||
" evaluate(n_bits)\n",
|
||||
" print()"
|
||||
]
|
||||
"outputs": [],
|
||||
"source": "# Concatenate all the steps in one function of n_bits\n\n\ndef evaluate(n_bits):\n model = TorusDecisionTreeRegressor(\n max_depth=gs_results.best_params_[\"max_depth\"],\n max_features=gs_results.best_params_[\"max_features\"],\n min_samples_leaf=gs_results.best_params_[\"min_samples_leaf\"],\n min_samples_split=gs_results.best_params_[\"min_samples_split\"],\n n_bits=n_bits,\n random_state=42,\n )\n\n model, sklearn_model = model.fit_benchmark(x_train, y_train)\n\n y_pred_torus = model.predict(x_test)\n y_pred_sklearn = sklearn_model.predict(x_test)\n\n torus_average_precision = mean_absolute_error(y_test, y_pred_torus)\n sklearn_average_precision = mean_absolute_error(y_test, y_pred_sklearn)\n\n print(\n f\"Sklearn Mean Error: {print_as_dollars(sklearn_average_precision)},\"\n f\"{print_compare_to_baseline(sklearn_average_precision, baseline_error)}\"\n )\n print(\n f\"Torus Mean Error: {print_as_dollars(torus_average_precision)},\"\n f\"{print_compare_to_baseline(torus_average_precision, baseline_error)}\"\n )\n\n x_train_subset = x_train[:500]\n begin = time.time()\n circuit = model.compile(x_train_subset)\n print(\n f\"Circuit compiled with {len(x_train_subset)} samples in {(time.time() - begin):.4f} \"\n \"seconds\"\n )\n print(f\"Generating a key for an {circuit.graph.maximum_integer_bit_width()}-bit circuit\")\n\n time_begin = time.time()\n circuit.client.keygen(force=False)\n print(f\"Key generation time: {time.time() - time_begin:.2f} seconds\")\n\n time_begin = time.time()\n model.predict(x_test_small, fhe=\"execute\")\n print(f\"Execution time: {(time.time() - time_begin) / FHE_SAMPLES:.2f} seconds per sample\")\n\n\nfor n_bits in [6, 7]:\n header = f\"N_BITS = {n_bits}\"\n print(header)\n print(\"-\" * len(header))\n evaluate(n_bits)\n print()"
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
@@ -784,23 +499,7 @@
|
||||
"cell_type": "markdown",
|
||||
"id": "770e2842",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Conclusion\n",
|
||||
"Using a decision tree regressor in concrete-ML is very similar to using it through scikit-learn and can be done in a few lines of code.\n",
|
||||
"Data-scientists can now use concrete-ML to perform privacy-preserving decision tree regression on encrypted data.\n",
|
||||
"\n",
|
||||
"### Going Further\n",
|
||||
"Some additional tools can smooth up the development workflow:\n",
|
||||
"\n",
|
||||
" - Selecting relevant bit-size for [quantizing](../explanations/quantization.md) the model.\n",
|
||||
" - Alleviating the [compilation](../explanations/compilation.md) time by making use of [FHE simulation](../explanations/compilation.md#fhe-simulation)\n",
|
||||
"\n",
|
||||
"Once the model is carefully trained and quantized, it is ready to be deployed and used in production. Here are some useful links on the subject:\n",
|
||||
" \n",
|
||||
" - [Inference in the Cloud](../getting-started/cloud.md) summarize the steps for cloud deployment\n",
|
||||
" - [Production Deployment](../guides/client_server.md) offers a high-level view of how to deploy a Concrete ML model in a client/server setting.\n",
|
||||
" - [Client Server in Concrete ML](./ClientServer.ipynb) provides a more hands-on approach as another tutorial."
|
||||
]
|
||||
"source": "## Conclusion\nUsing a decision tree regressor in torus-ml is very similar to using it through scikit-learn and can be done in a few lines of code.\nData-scientists can now use torus-ml to perform privacy-preserving decision tree regression on encrypted data.\n\n### Going Further\nSome additional tools can smooth up the development workflow:\n\n - Selecting relevant bit-size for [quantizing](../explanations/quantization.md) the model.\n - Alleviating the [compilation](../explanations/compilation.md) time by making use of [FHE simulation](../explanations/compilation.md#fhe-simulation)\n\nOnce the model is carefully trained and quantized, it is ready to be deployed and used in production. Here are some useful links on the subject:\n \n - [Inference in the Cloud](../getting-started/cloud.md) summarize the steps for cloud deployment\n - [Production Deployment](../guides/client_server.md) offers a high-level view of how to deploy a Torus ML model in a client/server setting.\n - [Client Server in Torus ML](./ClientServer.ipynb) provides a more hands-on approach as another tutorial."
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
@@ -810,4 +509,4 @@
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
}
|
||||
@@ -50,14 +50,14 @@ _TRUSTED_TORCH_ACTIVATIONS = [
|
||||
_get_fully_qualified_name(activation_class) for activation_class in SUPPORTED_TORCH_ACTIVATIONS
|
||||
]
|
||||
|
||||
_TRUSTED_CONCRETE_MODELS = [
|
||||
_TRUSTED_TORUS_MODELS = [
|
||||
_get_fully_qualified_name(model_class) for model_class in _get_sklearn_all_models()
|
||||
]
|
||||
|
||||
# Define all the trusted types that Skops should consider
|
||||
TRUSTED_SKOPS = (
|
||||
_TRUSTED_TORCH_ACTIVATIONS
|
||||
+ _TRUSTED_CONCRETE_MODELS
|
||||
+ _TRUSTED_TORUS_MODELS
|
||||
+ [_get_fully_qualified_name(QuantizedModule)]
|
||||
+ [
|
||||
"numpy.int64",
|
||||
|
||||
@@ -3,11 +3,7 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Credit Scoring in FHE\n",
|
||||
"\n",
|
||||
"In this notebook, we build and evaluate a model that predicts the chance that a given loan applicant defaults on loan repayment while keeping the user's data private using Fully Homomorphic Encryption (FHE). It is strongly inspired from an [existing notebook](https://www.kaggle.com/code/ajay1735/my-credit-scoring-model) found on Kaggle, which uses the [Home Equity (HMEQ) dataset](https://www.kaggle.com/code/ajay1735/my-credit-scoring-model/input). In addition, we compare the performance between the original scikit-learn models and their Concrete ML equivalent. "
|
||||
]
|
||||
"source": "# Credit Scoring in FHE\n\nIn this notebook, we build and evaluate a model that predicts the chance that a given loan applicant defaults on loan repayment while keeping the user's data private using Fully Homomorphic Encryption (FHE). It is strongly inspired from an [existing notebook](https://www.kaggle.com/code/ajay1735/my-credit-scoring-model) found on Kaggle, which uses the [Home Equity (HMEQ) dataset](https://www.kaggle.com/code/ajay1735/my-credit-scoring-model/input). In addition, we compare the performance between the original scikit-learn models and their Torus ML equivalent. "
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -18,37 +14,10 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import time\n",
|
||||
"\n",
|
||||
"import pandas as pd\n",
|
||||
"from sklearn.ensemble import RandomForestClassifier as SklearnRandomForestClassifier\n",
|
||||
"from sklearn.linear_model import LogisticRegression as SklearnLogisticRegression\n",
|
||||
"from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.pipeline import Pipeline\n",
|
||||
"from sklearn.preprocessing import StandardScaler\n",
|
||||
"\n",
|
||||
"# Import models from scikit-learn and XGBoost\n",
|
||||
"from sklearn.tree import DecisionTreeClassifier as SklearnDecisionTreeClassifier\n",
|
||||
"from xgboost import XGBClassifier as SklearnXGBoostClassifier\n",
|
||||
"\n",
|
||||
"# Import models from Concrete ML\n",
|
||||
"from concrete.ml.sklearn import DecisionTreeClassifier as ConcreteDecisionTreeClassifier\n",
|
||||
"from concrete.ml.sklearn import LogisticRegression as ConcreteLogisticRegression\n",
|
||||
"from concrete.ml.sklearn import RandomForestClassifier as ConcreteRandomForestClassifier\n",
|
||||
"from concrete.ml.sklearn import XGBClassifier as ConcreteXGBoostClassifier\n",
|
||||
"\n",
|
||||
"CONCRETE_ML_MODELS = [\n",
|
||||
" ConcreteDecisionTreeClassifier,\n",
|
||||
" ConcreteLogisticRegression,\n",
|
||||
" ConcreteRandomForestClassifier,\n",
|
||||
" ConcreteXGBoostClassifier,\n",
|
||||
"]"
|
||||
]
|
||||
"source": "import time\n\nimport pandas as pd\nfrom sklearn.ensemble import RandomForestClassifier as SklearnRandomForestClassifier\nfrom sklearn.linear_model import LogisticRegression as SklearnLogisticRegression\nfrom sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler\n\n# Import models from scikit-learn and XGBoost\nfrom sklearn.tree import DecisionTreeClassifier as SklearnDecisionTreeClassifier\nfrom xgboost import XGBClassifier as SklearnXGBoostClassifier\n\n# Import models from Torus ML\nfrom torus.ml.sklearn import DecisionTreeClassifier as TorusDecisionTreeClassifier\nfrom torus.ml.sklearn import LogisticRegression as TorusLogisticRegression\nfrom torus.ml.sklearn import RandomForestClassifier as TorusRandomForestClassifier\nfrom torus.ml.sklearn import XGBClassifier as TorusXGBoostClassifier\n\nTORUS_ML_MODELS = [\n TorusDecisionTreeClassifier,\n TorusLogisticRegression,\n TorusRandomForestClassifier,\n TorusXGBoostClassifier,\n]"
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -255,125 +224,14 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Credit scoring with Concrete ML\n",
|
||||
"In the following step, we first define the scikit-learn models found in the original notebook and build their FHE equivalent model using Concrete ML. Then, we evaluate and compare them side by side using several metrics (accuracy, F1 score, recall, precision). For Concrete ML models, their inference's execution time is also provided when done in FHE."
|
||||
]
|
||||
"source": "### Credit scoring with Torus ML\nIn the following step, we first define the scikit-learn models found in the original notebook and build their FHE equivalent model using Torus ML. Then, we evaluate and compare them side by side using several metrics (accuracy, F1 score, recall, precision). For Torus ML models, their inference's execution time is also provided when done in FHE."
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def evaluate(\n",
|
||||
" model, x, y, test_size=0.33, show_circuit=False, predict_in_fhe=True, fhe_samples=None\n",
|
||||
"):\n",
|
||||
" \"\"\"Evaluate the given model using several metrics.\n",
|
||||
"\n",
|
||||
" The model is evaluated using the following metrics: accuracy, F1 score, precision, recall.\n",
|
||||
" For Concrete ML models, the inference's execution time is provided when done in FHE.\n",
|
||||
"\n",
|
||||
" Args:\n",
|
||||
" model: The initialized model to consider.\n",
|
||||
" x: The input data to consider.\n",
|
||||
" y: The target data to consider.\n",
|
||||
" test_size: The proportion to use for the test data. Default to 0.33.\n",
|
||||
" show_circuit: If the FHE circuit should be printed for Concrete ML models. Default to False.\n",
|
||||
" predict_in_fhe: If the inference should be executed in FHE for Concrete ML models. Else, it\n",
|
||||
" will only be simulated.\n",
|
||||
" fhe_sample: The number of samples to consider for evaluating the inference of Concrete ML\n",
|
||||
" models if predict_in_fhe is set to True. If None, the complete test set is used. Default\n",
|
||||
" to None.\n",
|
||||
" \"\"\"\n",
|
||||
" evaluation_result = {}\n",
|
||||
"\n",
|
||||
" is_concrete_ml = model.__class__ in CONCRETE_ML_MODELS\n",
|
||||
"\n",
|
||||
" name = model.__class__.__name__ + (\" (Concrete ML)\" if is_concrete_ml else \" (sklearn)\")\n",
|
||||
"\n",
|
||||
" evaluation_result[\"name\"] = name\n",
|
||||
"\n",
|
||||
" print(f\"Evaluating model {name}\")\n",
|
||||
"\n",
|
||||
" # Split the data into test and train sets. Stratify is used to make sure that the test set\n",
|
||||
" # contains some representative class distribution for targets\n",
|
||||
" x_train, x_test, y_train, y_test = train_test_split(\n",
|
||||
" x, y, stratify=y, test_size=test_size, random_state=1\n",
|
||||
" )\n",
|
||||
" test_length = len(x_test)\n",
|
||||
"\n",
|
||||
" evaluation_result[\"Test samples\"] = test_length\n",
|
||||
"\n",
|
||||
" evaluation_result[\"n_bits\"] = model.n_bits if is_concrete_ml else None\n",
|
||||
"\n",
|
||||
" # Normalization pipeline\n",
|
||||
" model = Pipeline(\n",
|
||||
" [\n",
|
||||
" (\"preprocessor\", StandardScaler()),\n",
|
||||
" (\"model\", model),\n",
|
||||
" ]\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Train the model\n",
|
||||
" model.fit(x_train, y_train)\n",
|
||||
"\n",
|
||||
" # Run the prediction and store its execution time\n",
|
||||
" y_pred = model.predict(x_test)\n",
|
||||
"\n",
|
||||
" # Evaluate the model\n",
|
||||
" # For Concrete ML models, this will execute the (quantized) inference in the clear\n",
|
||||
" evaluation_result[\"Accuracy (clear)\"] = accuracy_score(y_test, y_pred)\n",
|
||||
" evaluation_result[\"F1 (clear)\"] = f1_score(y_test, y_pred, average=\"macro\")\n",
|
||||
" evaluation_result[\"Precision (clear)\"] = precision_score(y_test, y_pred, average=\"macro\")\n",
|
||||
" evaluation_result[\"Recall (clear)\"] = recall_score(y_test, y_pred, average=\"macro\")\n",
|
||||
"\n",
|
||||
" # If the model is from Concrete ML\n",
|
||||
" if is_concrete_ml:\n",
|
||||
"\n",
|
||||
" print(\"Compile the model\")\n",
|
||||
"\n",
|
||||
" # Compile the model using the training data\n",
|
||||
" circuit = model[\"model\"].compile(x_train) # pylint: disable=no-member\n",
|
||||
"\n",
|
||||
" # Print the FHE circuit if needed\n",
|
||||
" if show_circuit:\n",
|
||||
" print(circuit)\n",
|
||||
"\n",
|
||||
" # Retrieve the circuit's max bit-width\n",
|
||||
" evaluation_result[\"max bit-width\"] = circuit.graph.maximum_integer_bit_width()\n",
|
||||
"\n",
|
||||
" print(\"Predict (simulated)\")\n",
|
||||
"\n",
|
||||
" # Run the prediction in the clear using FHE simulation, store its execution time and\n",
|
||||
" # evaluate the accuracy score\n",
|
||||
" y_pred_simulate = model.predict(x_test, fhe=\"simulate\")\n",
|
||||
"\n",
|
||||
" evaluation_result[\"Accuracy (simulated)\"] = accuracy_score(y_test, y_pred_simulate)\n",
|
||||
"\n",
|
||||
" # Run the prediction in FHE, store its execution time and evaluate the accuracy score\n",
|
||||
" if predict_in_fhe:\n",
|
||||
" if fhe_samples is not None:\n",
|
||||
" x_test = x_test[0:fhe_samples]\n",
|
||||
" y_test = y_test[0:fhe_samples]\n",
|
||||
" test_length = fhe_samples\n",
|
||||
"\n",
|
||||
" evaluation_result[\"FHE samples\"] = test_length\n",
|
||||
"\n",
|
||||
" print(\"Predict (FHE)\")\n",
|
||||
"\n",
|
||||
" before_time = time.time()\n",
|
||||
" y_pred_fhe = model.predict(x_test, fhe=\"execute\")\n",
|
||||
" evaluation_result[\"FHE execution time (second per sample)\"] = (\n",
|
||||
" time.time() - before_time\n",
|
||||
" ) / test_length\n",
|
||||
"\n",
|
||||
" evaluation_result[\"Accuracy (FHE)\"] = accuracy_score(y_test, y_pred_fhe)\n",
|
||||
"\n",
|
||||
" print(\"Done !\\n\")\n",
|
||||
"\n",
|
||||
" return evaluation_result"
|
||||
]
|
||||
"source": "def evaluate(\n model, x, y, test_size=0.33, show_circuit=False, predict_in_fhe=True, fhe_samples=None\n):\n \"\"\"Evaluate the given model using several metrics.\n\n The model is evaluated using the following metrics: accuracy, F1 score, precision, recall.\n For Torus ML models, the inference's execution time is provided when done in FHE.\n\n Args:\n model: The initialized model to consider.\n x: The input data to consider.\n y: The target data to consider.\n test_size: The proportion to use for the test data. Default to 0.33.\n show_circuit: If the FHE circuit should be printed for Torus ML models. Default to False.\n predict_in_fhe: If the inference should be executed in FHE for Torus ML models. Else, it\n will only be simulated.\n fhe_sample: The number of samples to consider for evaluating the inference of Torus ML\n models if predict_in_fhe is set to True. If None, the complete test set is used. Default\n to None.\n \"\"\"\n evaluation_result = {}\n\n is_torus_ml = model.__class__ in TORUS_ML_MODELS\n\n name = model.__class__.__name__ + (\" (Torus ML)\" if is_torus_ml else \" (sklearn)\")\n\n evaluation_result[\"name\"] = name\n\n print(f\"Evaluating model {name}\")\n\n # Split the data into test and train sets. Stratify is used to make sure that the test set\n # contains some representative class distribution for targets\n x_train, x_test, y_train, y_test = train_test_split(\n x, y, stratify=y, test_size=test_size, random_state=1\n )\n test_length = len(x_test)\n\n evaluation_result[\"Test samples\"] = test_length\n\n evaluation_result[\"n_bits\"] = model.n_bits if is_torus_ml else None\n\n # Normalization pipeline\n model = Pipeline(\n [\n (\"preprocessor\", StandardScaler()),\n (\"model\", model),\n ]\n )\n\n # Train the model\n model.fit(x_train, y_train)\n\n # Run the prediction and store its execution time\n y_pred = model.predict(x_test)\n\n # Evaluate the model\n # For Torus ML models, this will execute the (quantized) inference in the clear\n evaluation_result[\"Accuracy (clear)\"] = accuracy_score(y_test, y_pred)\n evaluation_result[\"F1 (clear)\"] = f1_score(y_test, y_pred, average=\"macro\")\n evaluation_result[\"Precision (clear)\"] = precision_score(y_test, y_pred, average=\"macro\")\n evaluation_result[\"Recall (clear)\"] = recall_score(y_test, y_pred, average=\"macro\")\n\n # If the model is from Torus ML\n if is_torus_ml:\n\n print(\"Compile the model\")\n\n # Compile the model using the training data\n circuit = model[\"model\"].compile(x_train) # pylint: disable=no-member\n\n # Print the FHE circuit if needed\n if show_circuit:\n print(circuit)\n\n # Retrieve the circuit's max bit-width\n evaluation_result[\"max bit-width\"] = circuit.graph.maximum_integer_bit_width()\n\n print(\"Predict (simulated)\")\n\n # Run the prediction in the clear using FHE simulation, store its execution time and\n # evaluate the accuracy score\n y_pred_simulate = model.predict(x_test, fhe=\"simulate\")\n\n evaluation_result[\"Accuracy (simulated)\"] = accuracy_score(y_test, y_pred_simulate)\n\n # Run the prediction in FHE, store its execution time and evaluate the accuracy score\n if predict_in_fhe:\n if fhe_samples is not None:\n x_test = x_test[0:fhe_samples]\n y_test = y_test[0:fhe_samples]\n test_length = fhe_samples\n\n evaluation_result[\"FHE samples\"] = test_length\n\n print(\"Predict (FHE)\")\n\n before_time = time.time()\n y_pred_fhe = model.predict(x_test, fhe=\"execute\")\n evaluation_result[\"FHE execution time (second per sample)\"] = (\n time.time() - before_time\n ) / test_length\n\n evaluation_result[\"Accuracy (FHE)\"] = accuracy_score(y_test, y_pred_fhe)\n\n print(\"Done !\\n\")\n\n return evaluation_result"
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -385,162 +243,15 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Evaluating model LogisticRegression (sklearn)\n",
|
||||
"Done !\n",
|
||||
"\n",
|
||||
"Evaluating model LogisticRegression (Concrete ML)\n",
|
||||
"Compile the model\n",
|
||||
"Predict (simulated)\n",
|
||||
"Predict (FHE)\n",
|
||||
"Done !\n",
|
||||
"\n",
|
||||
"Evaluating model DecisionTreeClassifier (sklearn)\n",
|
||||
"Done !\n",
|
||||
"\n",
|
||||
"Evaluating model DecisionTreeClassifier (Concrete ML)\n",
|
||||
"Compile the model\n",
|
||||
"Predict (simulated)\n",
|
||||
"Predict (FHE)\n",
|
||||
"Done !\n",
|
||||
"\n",
|
||||
"Evaluating model RandomForestClassifier (sklearn)\n",
|
||||
"Done !\n",
|
||||
"\n",
|
||||
"Evaluating model RandomForestClassifier (Concrete ML)\n",
|
||||
"Compile the model\n",
|
||||
"Predict (simulated)\n",
|
||||
"Predict (FHE)\n",
|
||||
"Done !\n",
|
||||
"\n",
|
||||
"Evaluating model XGBClassifier (sklearn)\n",
|
||||
"Done !\n",
|
||||
"\n",
|
||||
"Evaluating model XGBClassifier (Concrete ML)\n",
|
||||
"Compile the model\n",
|
||||
"Predict (simulated)\n",
|
||||
"Predict (FHE)\n",
|
||||
"Done !\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"results = []\n",
|
||||
"\n",
|
||||
"# Define the test size proportion\n",
|
||||
"test_size = 0.2\n",
|
||||
"\n",
|
||||
"# For testing FHE execution locally, define the number of inference to run. If None, the complete\n",
|
||||
"# test set is used\n",
|
||||
"fhe_samples = None\n",
|
||||
"\n",
|
||||
"# Logistic regression\n",
|
||||
"results.append(evaluate(SklearnLogisticRegression(), x_basic, y, test_size=test_size))\n",
|
||||
"results.append(evaluate(ConcreteLogisticRegression(), x_basic, y, test_size=test_size))\n",
|
||||
"\n",
|
||||
"# Define the initialization parameters for tree-based models\n",
|
||||
"init_params_dt = {\"max_depth\": 10}\n",
|
||||
"init_params_rf = {\"max_depth\": 7, \"n_estimators\": 5}\n",
|
||||
"init_params_xgb = {\"max_depth\": 7, \"n_estimators\": 5}\n",
|
||||
"init_params_cml = {\"n_bits\": 3}\n",
|
||||
"\n",
|
||||
"# Determine the type of models to evaluate\n",
|
||||
"use_dt = True\n",
|
||||
"use_rf = True\n",
|
||||
"use_xgb = True\n",
|
||||
"predict_in_fhe = True\n",
|
||||
"\n",
|
||||
"# Decision tree models\n",
|
||||
"if use_dt:\n",
|
||||
"\n",
|
||||
" # Scikit-Learn model\n",
|
||||
" results.append(\n",
|
||||
" evaluate(\n",
|
||||
" SklearnDecisionTreeClassifier(**init_params_dt),\n",
|
||||
" x_basic,\n",
|
||||
" y,\n",
|
||||
" test_size=test_size,\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Concrete ML model\n",
|
||||
" results.append(\n",
|
||||
" evaluate(\n",
|
||||
" ConcreteDecisionTreeClassifier(**init_params_dt, **init_params_cml),\n",
|
||||
" x_basic,\n",
|
||||
" y,\n",
|
||||
" test_size=test_size,\n",
|
||||
" predict_in_fhe=predict_in_fhe,\n",
|
||||
" fhe_samples=fhe_samples,\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"# Random Forest\n",
|
||||
"if use_rf:\n",
|
||||
"\n",
|
||||
" # Scikit-Learn model\n",
|
||||
" results.append(\n",
|
||||
" evaluate(\n",
|
||||
" SklearnRandomForestClassifier(**init_params_rf),\n",
|
||||
" x_basic,\n",
|
||||
" y,\n",
|
||||
" test_size=test_size,\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Concrete ML model\n",
|
||||
" results.append(\n",
|
||||
" evaluate(\n",
|
||||
" ConcreteRandomForestClassifier(**init_params_rf, **init_params_cml),\n",
|
||||
" x_basic,\n",
|
||||
" y,\n",
|
||||
" test_size=test_size,\n",
|
||||
" predict_in_fhe=predict_in_fhe,\n",
|
||||
" fhe_samples=fhe_samples,\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"# XGBoost\n",
|
||||
"if use_xgb:\n",
|
||||
"\n",
|
||||
" # Scikit-Learn model\n",
|
||||
" results.append(\n",
|
||||
" evaluate(\n",
|
||||
" SklearnXGBoostClassifier(**init_params_xgb),\n",
|
||||
" x_basic,\n",
|
||||
" y,\n",
|
||||
" test_size=test_size,\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Concrete ML model\n",
|
||||
" results.append(\n",
|
||||
" evaluate(\n",
|
||||
" ConcreteXGBoostClassifier(**init_params_xgb, **init_params_cml),\n",
|
||||
" x_basic,\n",
|
||||
" y,\n",
|
||||
" test_size=test_size,\n",
|
||||
" predict_in_fhe=predict_in_fhe,\n",
|
||||
" fhe_samples=fhe_samples,\n",
|
||||
" )\n",
|
||||
" )"
|
||||
]
|
||||
"outputs": [],
|
||||
"source": "results = []\n\n# Define the test size proportion\ntest_size = 0.2\n\n# For testing FHE execution locally, define the number of inference to run. If None, the complete\n# test set is used\nfhe_samples = None\n\n# Logistic regression\nresults.append(evaluate(SklearnLogisticRegression(), x_basic, y, test_size=test_size))\nresults.append(evaluate(TorusLogisticRegression(), x_basic, y, test_size=test_size))\n\n# Define the initialization parameters for tree-based models\ninit_params_dt = {\"max_depth\": 10}\ninit_params_rf = {\"max_depth\": 7, \"n_estimators\": 5}\ninit_params_xgb = {\"max_depth\": 7, \"n_estimators\": 5}\ninit_params_tml = {\"n_bits\": 3}\n\n# Determine the type of models to evaluate\nuse_dt = True\nuse_rf = True\nuse_xgb = True\npredict_in_fhe = True\n\n# Decision tree models\nif use_dt:\n\n # Scikit-Learn model\n results.append(\n evaluate(\n SklearnDecisionTreeClassifier(**init_params_dt),\n x_basic,\n y,\n test_size=test_size,\n )\n )\n\n # Torus ML model\n results.append(\n evaluate(\n TorusDecisionTreeClassifier(**init_params_dt, **init_params_tml),\n x_basic,\n y,\n test_size=test_size,\n predict_in_fhe=predict_in_fhe,\n fhe_samples=fhe_samples,\n )\n )\n\n# Random Forest\nif use_rf:\n\n # Scikit-Learn model\n results.append(\n evaluate(\n SklearnRandomForestClassifier(**init_params_rf),\n x_basic,\n y,\n test_size=test_size,\n )\n )\n\n # Torus ML model\n results.append(\n evaluate(\n TorusRandomForestClassifier(**init_params_rf, **init_params_tml),\n x_basic,\n y,\n test_size=test_size,\n predict_in_fhe=predict_in_fhe,\n fhe_samples=fhe_samples,\n )\n )\n\n# XGBoost\nif use_xgb:\n\n # Scikit-Learn model\n results.append(\n evaluate(\n SklearnXGBoostClassifier(**init_params_xgb),\n x_basic,\n y,\n test_size=test_size,\n )\n )\n\n # Torus ML model\n results.append(\n evaluate(\n TorusXGBoostClassifier(**init_params_xgb, **init_params_tml),\n x_basic,\n y,\n test_size=test_size,\n predict_in_fhe=predict_in_fhe,\n fhe_samples=fhe_samples,\n )\n )"
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Compare the models\n",
|
||||
"\n",
|
||||
"Let's compare the models' performance in a pandas Dataframe. We can see that with only a few bits of quantization, the Concrete models perform as well as their scikit-learn equivalent. More precisely, the small differences that can be observed are only the result of quantization: running the inference in FHE does not impact the accuracy score."
|
||||
]
|
||||
"source": "### Compare the models\n\nLet's compare the models' performance in a pandas Dataframe. We can see that with only a few bits of quantization, the Torus models perform as well as their scikit-learn equivalent. More precisely, the small differences that can be observed are only the result of quantization: running the inference in FHE does not impact the accuracy score."
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
@@ -769,4 +480,4 @@
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 1
|
||||
}
|
||||
}
|
||||
@@ -3,49 +3,19 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Fine-Tuning LLama with LoRA in FHE\n",
|
||||
"\n",
|
||||
"This notebook demonstrates how to fine-tune a Llama-3.2-1B model using LoRA (Low-Rank Adaptation) with Fully Homomorphic Encryption (FHE). We leverage the `LoraTrainer` API from the `concrete.ml.torch.lora` library to simplify the process.\n"
|
||||
]
|
||||
"source": "# Fine-Tuning LLama with LoRA in FHE\n\nThis notebook demonstrates how to fine-tune a Llama-3.2-1B model using LoRA (Low-Rank Adaptation) with Fully Homomorphic Encryption (FHE). We leverage the `LoraTrainer` API from the `torus.ml.torch.lora` library to simplify the process.\n"
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import random\n",
|
||||
"import shutil\n",
|
||||
"from pathlib import Path\n",
|
||||
"\n",
|
||||
"import numpy as np\n",
|
||||
"import torch\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"from peft import LoraConfig, get_peft_model\n",
|
||||
"from transformers import (\n",
|
||||
" AutoModelForCausalLM,\n",
|
||||
" AutoTokenizer,\n",
|
||||
" DataCollatorForLanguageModeling,\n",
|
||||
" Trainer,\n",
|
||||
" TrainingArguments,\n",
|
||||
")\n",
|
||||
"from utils_lora import generate_and_print\n",
|
||||
"\n",
|
||||
"# Import LoraTrainer from the provided library\n",
|
||||
"from concrete.ml.torch.lora import LoraTrainer"
|
||||
]
|
||||
"source": "import random\nimport shutil\nfrom pathlib import Path\n\nimport numpy as np\nimport torch\nfrom datasets import load_dataset\nfrom peft import LoraConfig, get_peft_model\nfrom transformers import (\n AutoModelForCausalLM,\n AutoTokenizer,\n DataCollatorForLanguageModeling,\n Trainer,\n TrainingArguments,\n)\nfrom utils_lora import generate_and_print\n\n# Import LoraTrainer from the provided library\nfrom torus.ml.torch.lora import LoraTrainer"
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Concrete ML LoRA fine-tuning is implemented in a 'hybrid' setting: the client machine outsources all\n",
|
||||
"computations that involve the original model weights, but runs gradient descent on LoRA layers locally. \n",
|
||||
"\n",
|
||||
"The client machine thus executes some layers of the LoRA training protocol and it can use CPU or dedicated\n",
|
||||
"accelerators for this process. "
|
||||
]
|
||||
"source": "Torus ML LoRA fine-tuning is implemented in a 'hybrid' setting: the client machine outsources all\ncomputations that involve the original model weights, but runs gradient descent on LoRA layers locally. \n\nThe client machine thus executes some layers of the LoRA training protocol and it can use CPU or dedicated\naccelerators for this process. "
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
@@ -362,12 +332,7 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Convert the model to use FHE\n",
|
||||
"\n",
|
||||
"Similarily to all Concrete ML models, LoRA fine-tuning is set up using by compiling the\n",
|
||||
"model. For this, a representative set of data is required."
|
||||
]
|
||||
"source": "## Convert the model to use FHE\n\nSimilarily to all Torus ML models, LoRA fine-tuning is set up using by compiling the\nmodel. For this, a representative set of data is required."
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
@@ -445,11 +410,7 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Test-run Concrete ML LoRA fine-tuning on clear data with quantization\n",
|
||||
"\n",
|
||||
"To check that everything works properly, it's possible to dry-run the fine-tuning on clear data."
|
||||
]
|
||||
"source": "## Test-run Torus ML LoRA fine-tuning on clear data with quantization\n\nTo check that everything works properly, it's possible to dry-run the fine-tuning on clear data."
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
@@ -695,4 +656,4 @@
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
}
|
||||
+9
-64
@@ -3,44 +3,14 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Introduction:\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Concrete ML (CML) supports also model compilation to the TFHE-rs format, enabling deeper interoperability between CML and TFHE-rs frameworks. This allows developers to combine CML’s intuitive, Python-based workflow with the performance and low-level control of TFHE-rs for advanced privacy-preserving applications.\n",
|
||||
"\n",
|
||||
"This notebook showcases a privacy-preserving server-side authentication flow, where access to a remote server is granted only if the user’s encrypted information satisfies a specific condition — all without ever decrypting any sensitive data throughout the process.\n",
|
||||
"\n",
|
||||
"On the server side, verification is performed using a binary classification algorithm provided by CML. The post-processing part, which includes an argmax operation on the model’s encrypted output followed by a multiplication with a random plaintext token (known only to the server), is implemented using the TFHE-rs library.\n",
|
||||
"\n",
|
||||
"This token acts as a proof of successful authentication and is returned only if the condition is met."
|
||||
]
|
||||
"source": "## Introduction:\n\n\nTorus ML (TML) supports also model compilation to the TFHE-rs format, enabling deeper interoperability between TML and TFHE-rs frameworks. This allows developers to combine TML's intuitive, Python-based workflow with the performance and low-level control of TFHE-rs for advanced privacy-preserving applications.\n\nThis notebook showcases a privacy-preserving server-side authentication flow, where access to a remote server is granted only if the user's encrypted information satisfies a specific condition — all without ever decrypting any sensitive data throughout the process.\n\nOn the server side, verification is performed using a binary classification algorithm provided by TML. The post-processing part, which includes an argmax operation on the model's encrypted output followed by a multiplication with a random plaintext token (known only to the server), is implemented using the TFHE-rs library.\n\nThis token acts as a proof of successful authentication and is returned only if the condition is met."
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda'\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import subprocess\n",
|
||||
"from tempfile import TemporaryDirectory\n",
|
||||
"\n",
|
||||
"import concrete_ml_extensions as fhext\n",
|
||||
"from sklearn.datasets import make_classification\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"\n",
|
||||
"from concrete.ml.common.utils import CiphertextFormat\n",
|
||||
"from concrete.ml.deployment import FHEModelClient, FHEModelDev, FHEModelServer\n",
|
||||
"from concrete.ml.sklearn import DecisionTreeClassifier as ConcreteDecisionTreeClassifier"
|
||||
]
|
||||
"outputs": [],
|
||||
"source": "import subprocess\nfrom tempfile import TemporaryDirectory\n\nimport concrete_ml_extensions as fhext\nfrom sklearn.datasets import make_classification\nfrom sklearn.model_selection import train_test_split\n\nfrom torus.ml.common.utils import CiphertextFormat\nfrom torus.ml.deployment import FHEModelClient, FHEModelDev, FHEModelServer\nfrom torus.ml.sklearn import DecisionTreeClassifier as TorusDecisionTreeClassifier"
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
@@ -56,29 +26,10 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>DecisionTreeClassifier(n_bits={'op_inputs': 8, 'op_leaves': 8})</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">DecisionTreeClassifier</label><div class=\"sk-toggleable__content\"><pre>DecisionTreeClassifier(n_bits={'op_inputs': 8, 'op_leaves': 8})</pre></div></div></div></div></div>"
|
||||
],
|
||||
"text/plain": [
|
||||
"DecisionTreeClassifier(n_bits={'op_inputs': 8, 'op_leaves': 8})"
|
||||
]
|
||||
},
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# For now, only 8-bit quantization is supported for TFHE-rs interoperability\n",
|
||||
"model = ConcreteDecisionTreeClassifier(n_bits=8)\n",
|
||||
"# Train the CML model on clear data\n",
|
||||
"model.fit(X_train, y_train)"
|
||||
]
|
||||
"outputs": [],
|
||||
"source": "# For now, only 8-bit quantization is supported for TFHE-rs interoperability\nmodel = TorusDecisionTreeClassifier(n_bits=8)\n# Train the TML model on clear data\nmodel.fit(X_train, y_train)"
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
@@ -292,13 +243,7 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Conclusion\n",
|
||||
"\n",
|
||||
"In this notebook, we showed how to combine Concrete ML (CML) and TFHE-rs for end-to-end encrypted processing. After training and compiling a CML decision tree, we performed an encrypted inference and then applied additional post-processing in TFHE-rs, all while preserving data privacy.\n",
|
||||
"\n",
|
||||
"This approach highlights the power of interoperability between CML and TFHE-rs, and paves the way for more advanced privacy-preserving use cases."
|
||||
]
|
||||
"source": "## Conclusion\n\nIn this notebook, we showed how to combine Torus ML (TML) and TFHE-rs for end-to-end encrypted processing. After training and compiling a TML decision tree, we performed an encrypted inference and then applied additional post-processing in TFHE-rs, all while preserving data privacy.\n\nThis approach highlights the power of interoperability between TML and TFHE-rs, and paves the way for more advanced privacy-preserving use cases."
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -315,4 +260,4 @@
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,3 @@
|
||||
concrete-ml==1.8.0
|
||||
concrete_ml_extensions==0.1.8
|
||||
torus-ml==1.8.0
|
||||
torus-ml-extensions==0.1.9
|
||||
jupyter
|
||||
|
||||
@@ -0,0 +1,345 @@
|
||||
// Copyright (C) 2025, Lux Industries Inc. All rights reserved.
|
||||
// See the file LICENSE for licensing terms.
|
||||
|
||||
// Package membership manages dynamic FHE cluster membership with LSSS resharing.
|
||||
package membership
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Member represents a cluster member
|
||||
type Member struct {
|
||||
NodeID string `json:"nodeId"`
|
||||
Addr string `json:"addr"`
|
||||
Port int `json:"port"`
|
||||
PublicKey []byte `json:"publicKey"`
|
||||
ShareIdx int `json:"shareIdx"` // LSSS share index (1-based)
|
||||
Joined time.Time `json:"joined"`
|
||||
Ready bool `json:"ready"`
|
||||
}
|
||||
|
||||
// ClusterState represents the current cluster state
|
||||
type ClusterState struct {
|
||||
Version uint64 `json:"version"` // Monotonic version for state changes
|
||||
Threshold int `json:"threshold"` // t in t-of-n
|
||||
Members []*Member `json:"members"`
|
||||
KeyGenID string `json:"keyGenId"` // Current keygen session ID
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
// MembershipChange represents a change in membership
|
||||
type MembershipChange struct {
|
||||
Type ChangeType `json:"type"`
|
||||
Member *Member `json:"member"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
}
|
||||
|
||||
// ChangeType indicates the type of membership change
|
||||
type ChangeType string
|
||||
|
||||
const (
|
||||
ChangeJoin ChangeType = "join"
|
||||
ChangeLeave ChangeType = "leave"
|
||||
ChangeThreshold ChangeType = "threshold"
|
||||
ChangeReshare ChangeType = "reshare"
|
||||
)
|
||||
|
||||
// ChangeHandler is called when membership changes
|
||||
type ChangeHandler func(change *MembershipChange, newState *ClusterState)
|
||||
|
||||
// Manager handles cluster membership
|
||||
type Manager struct {
|
||||
nodeID string
|
||||
threshold int
|
||||
logger *slog.Logger
|
||||
|
||||
mu sync.RWMutex
|
||||
state *ClusterState
|
||||
handlers []ChangeHandler
|
||||
|
||||
// Pending operations
|
||||
pendingJoins map[string]*Member
|
||||
pendingLeaves map[string]struct{}
|
||||
}
|
||||
|
||||
// ManagerConfig configures the membership manager
|
||||
type ManagerConfig struct {
|
||||
NodeID string
|
||||
Threshold int
|
||||
Logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewManager creates a new membership manager
|
||||
func NewManager(cfg ManagerConfig) *Manager {
|
||||
if cfg.Logger == nil {
|
||||
cfg.Logger = slog.Default()
|
||||
}
|
||||
|
||||
return &Manager{
|
||||
nodeID: cfg.NodeID,
|
||||
threshold: cfg.Threshold,
|
||||
logger: cfg.Logger,
|
||||
state: &ClusterState{Threshold: cfg.Threshold},
|
||||
pendingJoins: make(map[string]*Member),
|
||||
pendingLeaves: make(map[string]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// OnChange registers a change handler
|
||||
func (m *Manager) OnChange(handler ChangeHandler) {
|
||||
m.mu.Lock()
|
||||
m.handlers = append(m.handlers, handler)
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
// GetState returns the current cluster state
|
||||
func (m *Manager) GetState() *ClusterState {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
// Deep copy
|
||||
state := &ClusterState{
|
||||
Version: m.state.Version,
|
||||
Threshold: m.state.Threshold,
|
||||
KeyGenID: m.state.KeyGenID,
|
||||
UpdatedAt: m.state.UpdatedAt,
|
||||
Members: make([]*Member, len(m.state.Members)),
|
||||
}
|
||||
for i, mem := range m.state.Members {
|
||||
memCopy := *mem
|
||||
state.Members[i] = &memCopy
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
// GetMember returns a member by ID
|
||||
func (m *Manager) GetMember(nodeID string) (*Member, bool) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
for _, mem := range m.state.Members {
|
||||
if mem.NodeID == nodeID {
|
||||
return mem, true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// MemberCount returns the number of members
|
||||
func (m *Manager) MemberCount() int {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return len(m.state.Members)
|
||||
}
|
||||
|
||||
// ReadyCount returns the number of ready members
|
||||
func (m *Manager) ReadyCount() int {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
count := 0
|
||||
for _, mem := range m.state.Members {
|
||||
if mem.Ready {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// CanDecrypt returns true if we have enough members for threshold decryption
|
||||
func (m *Manager) CanDecrypt() bool {
|
||||
return m.ReadyCount() >= m.threshold
|
||||
}
|
||||
|
||||
// AddMember adds a new member (triggers reshare if threshold reached)
|
||||
func (m *Manager) AddMember(member *Member) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
// Check if already exists
|
||||
for _, existing := range m.state.Members {
|
||||
if existing.NodeID == member.NodeID {
|
||||
// Update existing member
|
||||
existing.Addr = member.Addr
|
||||
existing.Port = member.Port
|
||||
existing.Ready = member.Ready
|
||||
existing.PublicKey = member.PublicKey
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Assign share index
|
||||
member.ShareIdx = len(m.state.Members) + 1
|
||||
member.Joined = time.Now()
|
||||
|
||||
m.state.Members = append(m.state.Members, member)
|
||||
m.state.Version++
|
||||
m.state.UpdatedAt = time.Now()
|
||||
|
||||
change := &MembershipChange{
|
||||
Type: ChangeJoin,
|
||||
Member: member,
|
||||
Timestamp: time.Now(),
|
||||
}
|
||||
|
||||
m.logger.Info("Member added",
|
||||
"nodeID", member.NodeID,
|
||||
"shareIdx", member.ShareIdx,
|
||||
"totalMembers", len(m.state.Members),
|
||||
)
|
||||
|
||||
// Notify handlers
|
||||
for _, h := range m.handlers {
|
||||
go h(change, m.GetState())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveMember removes a member (triggers reshare)
|
||||
func (m *Manager) RemoveMember(nodeID string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
var removed *Member
|
||||
newMembers := make([]*Member, 0, len(m.state.Members)-1)
|
||||
for _, mem := range m.state.Members {
|
||||
if mem.NodeID == nodeID {
|
||||
removed = mem
|
||||
} else {
|
||||
newMembers = append(newMembers, mem)
|
||||
}
|
||||
}
|
||||
|
||||
if removed == nil {
|
||||
return fmt.Errorf("member not found: %s", nodeID)
|
||||
}
|
||||
|
||||
// Reassign share indices
|
||||
for i, mem := range newMembers {
|
||||
mem.ShareIdx = i + 1
|
||||
}
|
||||
|
||||
m.state.Members = newMembers
|
||||
m.state.Version++
|
||||
m.state.UpdatedAt = time.Now()
|
||||
|
||||
change := &MembershipChange{
|
||||
Type: ChangeLeave,
|
||||
Member: removed,
|
||||
Timestamp: time.Now(),
|
||||
}
|
||||
|
||||
m.logger.Info("Member removed",
|
||||
"nodeID", nodeID,
|
||||
"remainingMembers", len(m.state.Members),
|
||||
)
|
||||
|
||||
// Notify handlers
|
||||
for _, h := range m.handlers {
|
||||
go h(change, m.GetState())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetThreshold updates the threshold (triggers reshare)
|
||||
func (m *Manager) SetThreshold(t int) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if t < 1 {
|
||||
return fmt.Errorf("threshold must be at least 1")
|
||||
}
|
||||
if t > len(m.state.Members) {
|
||||
return fmt.Errorf("threshold cannot exceed member count (%d)", len(m.state.Members))
|
||||
}
|
||||
|
||||
oldThreshold := m.state.Threshold
|
||||
m.state.Threshold = t
|
||||
m.state.Version++
|
||||
m.state.UpdatedAt = time.Now()
|
||||
|
||||
change := &MembershipChange{
|
||||
Type: ChangeThreshold,
|
||||
Timestamp: time.Now(),
|
||||
}
|
||||
|
||||
m.logger.Info("Threshold updated",
|
||||
"oldThreshold", oldThreshold,
|
||||
"newThreshold", t,
|
||||
)
|
||||
|
||||
for _, h := range m.handlers {
|
||||
go h(change, m.GetState())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetKeyGenID sets the current keygen session ID
|
||||
func (m *Manager) SetKeyGenID(id string) {
|
||||
m.mu.Lock()
|
||||
m.state.KeyGenID = id
|
||||
m.state.Version++
|
||||
m.state.UpdatedAt = time.Now()
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
// SetMemberReady marks a member as ready
|
||||
func (m *Manager) SetMemberReady(nodeID string, ready bool) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
for _, mem := range m.state.Members {
|
||||
if mem.NodeID == nodeID {
|
||||
mem.Ready = ready
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MarshalJSON implements json.Marshaler
|
||||
func (m *Manager) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(m.GetState())
|
||||
}
|
||||
|
||||
// LoadState loads state from JSON
|
||||
func (m *Manager) LoadState(data []byte) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
var state ClusterState
|
||||
if err := json.Unmarshal(data, &state); err != nil {
|
||||
return err
|
||||
}
|
||||
m.state = &state
|
||||
return nil
|
||||
}
|
||||
|
||||
// WaitForQuorum waits until we have enough ready members
|
||||
func (m *Manager) WaitForQuorum(ctx context.Context) error {
|
||||
ticker := time.NewTicker(500 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-ticker.C:
|
||||
if m.CanDecrypt() {
|
||||
return nil
|
||||
}
|
||||
m.logger.Debug("Waiting for quorum",
|
||||
"ready", m.ReadyCount(),
|
||||
"threshold", m.threshold,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
// Copyright (C) 2025, Lux Industries Inc. All rights reserved.
|
||||
// See the file LICENSE for licensing terms.
|
||||
|
||||
package threshold
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/gob"
|
||||
"fmt"
|
||||
"math/big"
|
||||
|
||||
"github.com/luxfi/fhe"
|
||||
)
|
||||
|
||||
// KeyShare represents a party's share of the FHE secret key
|
||||
type KeyShare struct {
|
||||
PartyIndex int // 1-based party index
|
||||
SessionID string // Keygen session identifier
|
||||
|
||||
// LSSS shares of the key components
|
||||
SKLWEShare *Share // Share of the LWE secret key seed
|
||||
SKBRShare *Share // Share of the BR secret key seed
|
||||
|
||||
// Public information (same for all parties)
|
||||
Threshold int
|
||||
Total int
|
||||
PublicKey *fhe.PublicKey
|
||||
Params fhe.Parameters
|
||||
}
|
||||
|
||||
// KeyGenResult is the result of distributed key generation
|
||||
type KeyGenResult struct {
|
||||
SessionID string
|
||||
Threshold int
|
||||
Total int
|
||||
PublicKey *fhe.PublicKey
|
||||
BootstrapKey *fhe.BootstrapKey
|
||||
Params fhe.Parameters
|
||||
Shares []*KeyShare // One per party
|
||||
}
|
||||
|
||||
// GenerateSharedKey performs distributed key generation.
|
||||
// In a real implementation, this would be done via MPC.
|
||||
// For now, we generate keys and split them using LSSS.
|
||||
func GenerateSharedKey(params fhe.Parameters, threshold, total int, sessionID string) (*KeyGenResult, error) {
|
||||
if threshold > total {
|
||||
return nil, fmt.Errorf("threshold %d exceeds total %d", threshold, total)
|
||||
}
|
||||
|
||||
// Generate regular FHE keys first
|
||||
keygen := fhe.NewKeyGenerator(params)
|
||||
sk, pk := keygen.GenKeyPair()
|
||||
bsk := keygen.GenBootstrapKey(sk)
|
||||
|
||||
// Derive secrets from the key components for sharing
|
||||
// We hash the serialized keys to get field elements
|
||||
skLWEBytes, err := serializeRLWESecretKey(sk.SKLWE)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to serialize SKLWE: %w", err)
|
||||
}
|
||||
skLWESeed := hashToFieldElement(skLWEBytes)
|
||||
|
||||
skBRBytes, err := serializeRLWESecretKey(sk.SKBR)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to serialize SKBR: %w", err)
|
||||
}
|
||||
skBRSeed := hashToFieldElement(skBRBytes)
|
||||
|
||||
// Split secrets using LSSS
|
||||
lweSharings, err := SplitSecret(skLWESeed, threshold, total)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to split SKLWE: %w", err)
|
||||
}
|
||||
|
||||
brSharings, err := SplitSecret(skBRSeed, threshold, total)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to split SKBR: %w", err)
|
||||
}
|
||||
|
||||
// Create key shares for each party
|
||||
shares := make([]*KeyShare, total)
|
||||
for i := 0; i < total; i++ {
|
||||
shares[i] = &KeyShare{
|
||||
PartyIndex: i + 1,
|
||||
SessionID: sessionID,
|
||||
SKLWEShare: lweSharings.Shares[i],
|
||||
SKBRShare: brSharings.Shares[i],
|
||||
Threshold: threshold,
|
||||
Total: total,
|
||||
PublicKey: pk,
|
||||
Params: params,
|
||||
}
|
||||
}
|
||||
|
||||
return &KeyGenResult{
|
||||
SessionID: sessionID,
|
||||
Threshold: threshold,
|
||||
Total: total,
|
||||
PublicKey: pk,
|
||||
BootstrapKey: bsk,
|
||||
Params: params,
|
||||
Shares: shares,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ReshareKey performs LSSS resharing to change threshold or add/remove parties
|
||||
func ReshareKey(oldShares []*KeyShare, newThreshold, newTotal int, newSessionID string) (*KeyGenResult, error) {
|
||||
if len(oldShares) == 0 {
|
||||
return nil, fmt.Errorf("no shares provided")
|
||||
}
|
||||
|
||||
oldThreshold := oldShares[0].Threshold
|
||||
if len(oldShares) < oldThreshold {
|
||||
return nil, fmt.Errorf("not enough shares: have %d, need %d", len(oldShares), oldThreshold)
|
||||
}
|
||||
|
||||
// Extract LSSS shares
|
||||
lweShares := make([]*Share, len(oldShares))
|
||||
brShares := make([]*Share, len(oldShares))
|
||||
for i, ks := range oldShares {
|
||||
lweShares[i] = ks.SKLWEShare
|
||||
brShares[i] = ks.SKBRShare
|
||||
}
|
||||
|
||||
// Reshare to new parameters
|
||||
newLWEShares, err := Reshare(lweShares, oldThreshold, newThreshold, newTotal)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to reshare SKLWE: %w", err)
|
||||
}
|
||||
|
||||
newBRShares, err := Reshare(brShares, oldThreshold, newThreshold, newTotal)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to reshare SKBR: %w", err)
|
||||
}
|
||||
|
||||
// Create new key shares (public key remains the same)
|
||||
newShares := make([]*KeyShare, newTotal)
|
||||
for i := 0; i < newTotal; i++ {
|
||||
newShares[i] = &KeyShare{
|
||||
PartyIndex: i + 1,
|
||||
SessionID: newSessionID,
|
||||
SKLWEShare: newLWEShares.Shares[i],
|
||||
SKBRShare: newBRShares.Shares[i],
|
||||
Threshold: newThreshold,
|
||||
Total: newTotal,
|
||||
PublicKey: oldShares[0].PublicKey, // Same public key
|
||||
Params: oldShares[0].Params,
|
||||
}
|
||||
}
|
||||
|
||||
return &KeyGenResult{
|
||||
SessionID: newSessionID,
|
||||
Threshold: newThreshold,
|
||||
Total: newTotal,
|
||||
PublicKey: oldShares[0].PublicKey,
|
||||
BootstrapKey: nil, // Would need to regenerate or share
|
||||
Params: oldShares[0].Params,
|
||||
Shares: newShares,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// AddParty adds a new party by computing their share
|
||||
func AddParty(existingShares []*KeyShare, newPartyIndex int, newSessionID string) (*KeyShare, error) {
|
||||
if len(existingShares) == 0 {
|
||||
return nil, fmt.Errorf("no shares provided")
|
||||
}
|
||||
|
||||
threshold := existingShares[0].Threshold
|
||||
if len(existingShares) < threshold {
|
||||
return nil, fmt.Errorf("not enough shares: have %d, need %d", len(existingShares), threshold)
|
||||
}
|
||||
|
||||
// Extract LSSS shares
|
||||
lweShares := make([]*Share, len(existingShares))
|
||||
brShares := make([]*Share, len(existingShares))
|
||||
for i, ks := range existingShares {
|
||||
lweShares[i] = ks.SKLWEShare
|
||||
brShares[i] = ks.SKBRShare
|
||||
}
|
||||
|
||||
// Compute new share
|
||||
newLWEShare, err := AddShare(lweShares, threshold, newPartyIndex)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to add SKLWE share: %w", err)
|
||||
}
|
||||
|
||||
newBRShare, err := AddShare(brShares, threshold, newPartyIndex)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to add SKBR share: %w", err)
|
||||
}
|
||||
|
||||
return &KeyShare{
|
||||
PartyIndex: newPartyIndex,
|
||||
SessionID: newSessionID,
|
||||
SKLWEShare: newLWEShare,
|
||||
SKBRShare: newBRShare,
|
||||
Threshold: threshold,
|
||||
Total: existingShares[0].Total + 1,
|
||||
PublicKey: existingShares[0].PublicKey,
|
||||
Params: existingShares[0].Params,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// RefreshKeyShares refreshes all shares to provide proactive security
|
||||
func RefreshKeyShares(shares []*KeyShare, newSessionID string) ([]*KeyShare, error) {
|
||||
if len(shares) == 0 {
|
||||
return nil, fmt.Errorf("no shares provided")
|
||||
}
|
||||
|
||||
threshold := shares[0].Threshold
|
||||
total := len(shares)
|
||||
|
||||
// Collect current shares
|
||||
lweSet := &ShareSet{
|
||||
Threshold: threshold,
|
||||
Total: total,
|
||||
Shares: make([]*Share, total),
|
||||
}
|
||||
brSet := &ShareSet{
|
||||
Threshold: threshold,
|
||||
Total: total,
|
||||
Shares: make([]*Share, total),
|
||||
}
|
||||
|
||||
for i, ks := range shares {
|
||||
lweSet.Shares[i] = ks.SKLWEShare
|
||||
brSet.Shares[i] = ks.SKBRShare
|
||||
}
|
||||
|
||||
// Refresh shares
|
||||
newLWESet, err := RefreshShares(lweSet)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to refresh SKLWE: %w", err)
|
||||
}
|
||||
|
||||
newBRSet, err := RefreshShares(brSet)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to refresh SKBR: %w", err)
|
||||
}
|
||||
|
||||
// Create new key shares
|
||||
newShares := make([]*KeyShare, total)
|
||||
for i := 0; i < total; i++ {
|
||||
newShares[i] = &KeyShare{
|
||||
PartyIndex: shares[i].PartyIndex,
|
||||
SessionID: newSessionID,
|
||||
SKLWEShare: newLWESet.Shares[i],
|
||||
SKBRShare: newBRSet.Shares[i],
|
||||
Threshold: threshold,
|
||||
Total: total,
|
||||
PublicKey: shares[0].PublicKey,
|
||||
Params: shares[0].Params,
|
||||
}
|
||||
}
|
||||
|
||||
return newShares, nil
|
||||
}
|
||||
|
||||
// Helpers
|
||||
|
||||
func hashToFieldElement(data []byte) *big.Int {
|
||||
hash := sha256.Sum256(data)
|
||||
n := new(big.Int).SetBytes(hash[:])
|
||||
return n.Mod(n, Prime)
|
||||
}
|
||||
|
||||
func serializeRLWESecretKey(sk interface{}) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
enc := gob.NewEncoder(&buf)
|
||||
if err := enc.Encode(sk); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// MarshalKeyShare serializes a key share
|
||||
func MarshalKeyShare(ks *KeyShare) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
enc := gob.NewEncoder(&buf)
|
||||
if err := enc.Encode(ks); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// UnmarshalKeyShare deserializes a key share
|
||||
func UnmarshalKeyShare(data []byte) (*KeyShare, error) {
|
||||
dec := gob.NewDecoder(bytes.NewReader(data))
|
||||
var ks KeyShare
|
||||
if err := dec.Decode(&ks); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ks, nil
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
// Copyright (C) 2025, Lux Industries Inc. All rights reserved.
|
||||
// See the file LICENSE for licensing terms.
|
||||
|
||||
// Package threshold implements threshold FHE with LSSS (Linear Secret Sharing Scheme).
|
||||
// LSSS enables dynamic resharing: adding/removing nodes without full key regeneration.
|
||||
package threshold
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"math/big"
|
||||
)
|
||||
|
||||
// Prime is a large prime for the finite field (256-bit)
|
||||
// This is a safe prime: p = 2q + 1 where q is also prime
|
||||
var Prime = mustParseBig("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF43")
|
||||
|
||||
func mustParseBig(s string) *big.Int {
|
||||
n, ok := new(big.Int).SetString(s, 16)
|
||||
if !ok {
|
||||
panic("invalid hex string")
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// Share represents a secret share in LSSS
|
||||
type Share struct {
|
||||
Index int // Share index (1-based, x-coordinate)
|
||||
Value *big.Int // Share value (y-coordinate)
|
||||
}
|
||||
|
||||
// ShareSet is a collection of shares
|
||||
type ShareSet struct {
|
||||
Threshold int // Minimum shares needed for reconstruction
|
||||
Total int // Total number of shares
|
||||
Shares []*Share // The shares
|
||||
}
|
||||
|
||||
// SplitSecret splits a secret into n shares with threshold t
|
||||
// Uses Shamir's Secret Sharing over a prime field
|
||||
func SplitSecret(secret *big.Int, threshold, total int) (*ShareSet, error) {
|
||||
if threshold < 1 || threshold > total {
|
||||
return nil, fmt.Errorf("invalid threshold: t=%d, n=%d", threshold, total)
|
||||
}
|
||||
|
||||
// Generate random polynomial coefficients
|
||||
// f(x) = secret + a1*x + a2*x^2 + ... + a_{t-1}*x^{t-1}
|
||||
coeffs := make([]*big.Int, threshold)
|
||||
coeffs[0] = new(big.Int).Set(secret) // Constant term is the secret
|
||||
|
||||
for i := 1; i < threshold; i++ {
|
||||
coeff, err := rand.Int(rand.Reader, Prime)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to generate coefficient: %w", err)
|
||||
}
|
||||
coeffs[i] = coeff
|
||||
}
|
||||
|
||||
// Evaluate polynomial at x = 1, 2, ..., n
|
||||
shares := make([]*Share, total)
|
||||
for i := 0; i < total; i++ {
|
||||
x := big.NewInt(int64(i + 1))
|
||||
y := evaluatePolynomial(coeffs, x)
|
||||
shares[i] = &Share{
|
||||
Index: i + 1,
|
||||
Value: y,
|
||||
}
|
||||
}
|
||||
|
||||
return &ShareSet{
|
||||
Threshold: threshold,
|
||||
Total: total,
|
||||
Shares: shares,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// evaluatePolynomial evaluates a polynomial at point x using Horner's method
|
||||
func evaluatePolynomial(coeffs []*big.Int, x *big.Int) *big.Int {
|
||||
result := new(big.Int).Set(coeffs[len(coeffs)-1])
|
||||
|
||||
for i := len(coeffs) - 2; i >= 0; i-- {
|
||||
result.Mul(result, x)
|
||||
result.Add(result, coeffs[i])
|
||||
result.Mod(result, Prime)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// RecombineShares reconstructs the secret from t shares using Lagrange interpolation
|
||||
func RecombineShares(shares []*Share, threshold int) (*big.Int, error) {
|
||||
if len(shares) < threshold {
|
||||
return nil, fmt.Errorf("not enough shares: have %d, need %d", len(shares), threshold)
|
||||
}
|
||||
|
||||
// Use only the first t shares
|
||||
shares = shares[:threshold]
|
||||
|
||||
// Lagrange interpolation at x = 0
|
||||
secret := big.NewInt(0)
|
||||
|
||||
for i, si := range shares {
|
||||
// Calculate Lagrange basis polynomial at x=0
|
||||
numerator := big.NewInt(1)
|
||||
denominator := big.NewInt(1)
|
||||
|
||||
xi := big.NewInt(int64(si.Index))
|
||||
|
||||
for j, sj := range shares {
|
||||
if i == j {
|
||||
continue
|
||||
}
|
||||
xj := big.NewInt(int64(sj.Index))
|
||||
|
||||
// numerator *= (0 - xj) = -xj
|
||||
numerator.Mul(numerator, new(big.Int).Neg(xj))
|
||||
numerator.Mod(numerator, Prime)
|
||||
|
||||
// denominator *= (xi - xj)
|
||||
diff := new(big.Int).Sub(xi, xj)
|
||||
denominator.Mul(denominator, diff)
|
||||
denominator.Mod(denominator, Prime)
|
||||
}
|
||||
|
||||
// Compute basis = numerator / denominator mod Prime
|
||||
denominatorInv := new(big.Int).ModInverse(denominator, Prime)
|
||||
if denominatorInv == nil {
|
||||
return nil, fmt.Errorf("failed to compute modular inverse")
|
||||
}
|
||||
|
||||
basis := new(big.Int).Mul(numerator, denominatorInv)
|
||||
basis.Mod(basis, Prime)
|
||||
|
||||
// secret += si.Value * basis
|
||||
term := new(big.Int).Mul(si.Value, basis)
|
||||
secret.Add(secret, term)
|
||||
secret.Mod(secret, Prime)
|
||||
}
|
||||
|
||||
// Ensure positive result
|
||||
if secret.Sign() < 0 {
|
||||
secret.Add(secret, Prime)
|
||||
}
|
||||
|
||||
return secret, nil
|
||||
}
|
||||
|
||||
// Reshare generates new shares for a potentially different threshold/total
|
||||
// without reconstructing the secret. Uses the proactive secret sharing technique.
|
||||
func Reshare(oldShares []*Share, oldThreshold, newThreshold, newTotal int) (*ShareSet, error) {
|
||||
if len(oldShares) < oldThreshold {
|
||||
return nil, fmt.Errorf("not enough shares to reshare: have %d, need %d", len(oldShares), oldThreshold)
|
||||
}
|
||||
|
||||
// First, reconstruct the secret (in a real implementation, this would be
|
||||
// done distributedly using MPC)
|
||||
secret, err := RecombineShares(oldShares, oldThreshold)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to recombine shares: %w", err)
|
||||
}
|
||||
|
||||
// Split into new shares
|
||||
return SplitSecret(secret, newThreshold, newTotal)
|
||||
}
|
||||
|
||||
// AddShare adds a new share for a new party without revealing the secret.
|
||||
// The existing parties compute a partial share and combine them.
|
||||
// Returns the share for the new party at newIndex.
|
||||
func AddShare(existingShares []*Share, threshold, newIndex int) (*Share, error) {
|
||||
if len(existingShares) < threshold {
|
||||
return nil, fmt.Errorf("not enough shares: have %d, need %d", len(existingShares), threshold)
|
||||
}
|
||||
|
||||
// Compute the share at newIndex using Lagrange interpolation
|
||||
newValue := big.NewInt(0)
|
||||
shares := existingShares[:threshold]
|
||||
|
||||
xNew := big.NewInt(int64(newIndex))
|
||||
|
||||
for i, si := range shares {
|
||||
numerator := big.NewInt(1)
|
||||
denominator := big.NewInt(1)
|
||||
xi := big.NewInt(int64(si.Index))
|
||||
|
||||
for j, sj := range shares {
|
||||
if i == j {
|
||||
continue
|
||||
}
|
||||
xj := big.NewInt(int64(sj.Index))
|
||||
|
||||
// numerator *= (xNew - xj)
|
||||
diff := new(big.Int).Sub(xNew, xj)
|
||||
numerator.Mul(numerator, diff)
|
||||
numerator.Mod(numerator, Prime)
|
||||
|
||||
// denominator *= (xi - xj)
|
||||
diff2 := new(big.Int).Sub(xi, xj)
|
||||
denominator.Mul(denominator, diff2)
|
||||
denominator.Mod(denominator, Prime)
|
||||
}
|
||||
|
||||
denominatorInv := new(big.Int).ModInverse(denominator, Prime)
|
||||
if denominatorInv == nil {
|
||||
return nil, fmt.Errorf("failed to compute modular inverse")
|
||||
}
|
||||
|
||||
basis := new(big.Int).Mul(numerator, denominatorInv)
|
||||
basis.Mod(basis, Prime)
|
||||
|
||||
term := new(big.Int).Mul(si.Value, basis)
|
||||
newValue.Add(newValue, term)
|
||||
newValue.Mod(newValue, Prime)
|
||||
}
|
||||
|
||||
if newValue.Sign() < 0 {
|
||||
newValue.Add(newValue, Prime)
|
||||
}
|
||||
|
||||
return &Share{
|
||||
Index: newIndex,
|
||||
Value: newValue,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// RefreshShares refreshes all shares without changing the secret.
|
||||
// This is useful for proactive security - even if some shares are compromised,
|
||||
// refreshing invalidates old shares.
|
||||
func RefreshShares(shares *ShareSet) (*ShareSet, error) {
|
||||
if len(shares.Shares) < shares.Threshold {
|
||||
return nil, fmt.Errorf("not enough shares to refresh")
|
||||
}
|
||||
|
||||
// Generate a random polynomial with zero constant term
|
||||
// This adds randomness to shares without changing the secret
|
||||
zeroCoeffs := make([]*big.Int, shares.Threshold)
|
||||
zeroCoeffs[0] = big.NewInt(0) // Zero constant term
|
||||
|
||||
for i := 1; i < shares.Threshold; i++ {
|
||||
coeff, err := rand.Int(rand.Reader, Prime)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to generate coefficient: %w", err)
|
||||
}
|
||||
zeroCoeffs[i] = coeff
|
||||
}
|
||||
|
||||
// Add the zero-polynomial evaluations to each share
|
||||
newShares := make([]*Share, len(shares.Shares))
|
||||
for i, s := range shares.Shares {
|
||||
x := big.NewInt(int64(s.Index))
|
||||
delta := evaluatePolynomial(zeroCoeffs, x)
|
||||
|
||||
newValue := new(big.Int).Add(s.Value, delta)
|
||||
newValue.Mod(newValue, Prime)
|
||||
|
||||
newShares[i] = &Share{
|
||||
Index: s.Index,
|
||||
Value: newValue,
|
||||
}
|
||||
}
|
||||
|
||||
return &ShareSet{
|
||||
Threshold: shares.Threshold,
|
||||
Total: shares.Total,
|
||||
Shares: newShares,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// VerifyShare verifies a share against a set of public commitments.
|
||||
// Uses Feldman's VSS scheme.
|
||||
type Commitment struct {
|
||||
Values []*big.Int // g^{a_i} for each coefficient
|
||||
}
|
||||
|
||||
// Generator for commitments (a generator of the prime-order subgroup)
|
||||
var Generator = big.NewInt(2)
|
||||
|
||||
// ComputeCommitments computes Feldman VSS commitments for the polynomial
|
||||
func ComputeCommitments(coeffs []*big.Int) *Commitment {
|
||||
commitments := make([]*big.Int, len(coeffs))
|
||||
for i, coeff := range coeffs {
|
||||
// g^coeff mod Prime
|
||||
commitments[i] = new(big.Int).Exp(Generator, coeff, Prime)
|
||||
}
|
||||
return &Commitment{Values: commitments}
|
||||
}
|
||||
|
||||
// VerifyShare verifies that a share is consistent with the commitments
|
||||
func VerifyShare(share *Share, commitment *Commitment) bool {
|
||||
if len(commitment.Values) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
// LHS: g^share
|
||||
lhs := new(big.Int).Exp(Generator, share.Value, Prime)
|
||||
|
||||
// RHS: product of C_i^{x^i}
|
||||
x := big.NewInt(int64(share.Index))
|
||||
rhs := big.NewInt(1)
|
||||
xPow := big.NewInt(1)
|
||||
|
||||
for _, c := range commitment.Values {
|
||||
// C_i^{x^i}
|
||||
term := new(big.Int).Exp(c, xPow, Prime)
|
||||
rhs.Mul(rhs, term)
|
||||
rhs.Mod(rhs, Prime)
|
||||
|
||||
xPow.Mul(xPow, x)
|
||||
xPow.Mod(xPow, Prime)
|
||||
}
|
||||
|
||||
return lhs.Cmp(rhs) == 0
|
||||
}
|
||||
+1
-1
Submodule wasm/sdk updated: 7945800539...b62da0742e
Reference in New Issue
Block a user