mirror of
https://github.com/zenlm/agent-sanitizer.git
synced 2026-07-26 22:31:10 +00:00
Initial release: Agent Sanitizer v0.1.0
A standalone tool to sanitize AI agent logs for safe dataset sharing. Features: - Clean credentials, PII, and crypto keys from agent logs - Support for agent, Cursor, Continue, Aider - Interactive CLI with rich formatting - Auto-upload to HuggingFace Hub - Dry-run mode for safety - Run with uvx (no installation needed) Usage: uvx agent-sanitizer uvx agent-sanitizer --input ~/.claude/projects --upload user/dataset Perfect for the community to share coding datasets safely!
This commit is contained in:
Hanzo Dev
+40
@@ -0,0 +1,40 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
*.egg
|
||||
|
||||
# Virtual environments
|
||||
venv/
|
||||
env/
|
||||
ENV/
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# Testing
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
htmlcov/
|
||||
|
||||
# Distribution
|
||||
*.whl
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
|
||||
# Output
|
||||
sanitized-dataset/
|
||||
clean-dataset/
|
||||
*.backup.*
|
||||
@@ -0,0 +1,28 @@
|
||||
BSD 3-Clause License
|
||||
|
||||
Copyright (c) 2025, Zen AI
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
3. Neither the name of the copyright holder nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
@@ -0,0 +1,269 @@
|
||||
# 🧹 Agent Sanitizer
|
||||
|
||||
**Clean AI agent logs for safe dataset sharing**
|
||||
|
||||
Turn your Claude Code, Cursor, Continue, or Aider logs into publishable training datasets with one command.
|
||||
|
||||
[](https://pypi.org/project/agent-sanitizer/)
|
||||
[](LICENSE)
|
||||
|
||||
## Quick Start
|
||||
|
||||
No installation required! Run with `uvx`:
|
||||
|
||||
```bash
|
||||
# Interactive mode (recommended for first time)
|
||||
uvx agent-sanitizer
|
||||
|
||||
# Specify input/output
|
||||
uvx agent-sanitizer --input ~/.claude/projects --output ./my-dataset
|
||||
|
||||
# Dry run to see what would change
|
||||
uvx agent-sanitizer --dry-run
|
||||
|
||||
# Upload directly to HuggingFace
|
||||
uvx agent-sanitizer --upload username/my-dataset
|
||||
```
|
||||
|
||||
## What It Does
|
||||
|
||||
Agent Sanitizer automatically removes:
|
||||
|
||||
- ✅ **Credentials**: API keys, passwords, tokens
|
||||
- ✅ **PII**: Names, emails, phone numbers
|
||||
- ✅ **Crypto**: Wallet keys, seed phrases
|
||||
- ✅ **Paths**: Identifying file paths and project names
|
||||
|
||||
While preserving:
|
||||
|
||||
- ✅ **Code examples**: Including test data and development configs
|
||||
- ✅ **Tool usage**: File operations, commands, workflows
|
||||
- ✅ **Thinking traces**: Multi-step reasoning
|
||||
- ✅ **Context**: Real coding patterns
|
||||
|
||||
## Supported Agents
|
||||
|
||||
- 🤖 **Claude Code** (`.claude/projects`)
|
||||
- 🔮 **Cursor** (`.cursor/logs`)
|
||||
- ⏭️ **Continue** (`.continue/sessions`)
|
||||
- 🔧 **Aider** (`.aider/history`)
|
||||
- 📝 **Any JSONL-based agent logs**
|
||||
|
||||
## Features
|
||||
|
||||
### 🔒 Comprehensive Security Audit
|
||||
|
||||
Before cleaning, scans for:
|
||||
- Credentials (API keys, passwords, tokens)
|
||||
- Personal information (names, emails, phones)
|
||||
- Cryptocurrency (wallets, keys, seeds)
|
||||
- Sensitive data (SSNs, private keys)
|
||||
|
||||
### 🎯 Smart Cleaning
|
||||
|
||||
- Pattern-based detection with low false positives
|
||||
- Context-aware replacements
|
||||
- Preserves test data and examples
|
||||
- Maintains dataset value
|
||||
|
||||
### 📤 HuggingFace Integration
|
||||
|
||||
Upload directly after sanitization:
|
||||
|
||||
```bash
|
||||
uvx agent-sanitizer \
|
||||
--input ~/.claude/projects \
|
||||
--upload username/my-coding-dataset \
|
||||
--private # optional: make dataset private
|
||||
```
|
||||
|
||||
### 🔍 Dry Run Mode
|
||||
|
||||
See exactly what would change:
|
||||
|
||||
```bash
|
||||
uvx agent-sanitizer --dry-run
|
||||
```
|
||||
|
||||
## Installation
|
||||
|
||||
### Option 1: Run with uvx (Recommended)
|
||||
|
||||
No installation needed:
|
||||
|
||||
```bash
|
||||
uvx agent-sanitizer
|
||||
```
|
||||
|
||||
### Option 2: Install with pip
|
||||
|
||||
```bash
|
||||
pip install agent-sanitizer
|
||||
agent-sanitizer --help
|
||||
```
|
||||
|
||||
### Option 3: Install from source
|
||||
|
||||
```bash
|
||||
git clone https://github.com/zenlm/agent-sanitizer
|
||||
cd agent-sanitizer
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```bash
|
||||
# Interactive - walks you through the process
|
||||
uvx agent-sanitizer
|
||||
|
||||
# Specify directories
|
||||
uvx agent-sanitizer \
|
||||
--input ~/.claude/projects \
|
||||
--output ./clean-dataset
|
||||
|
||||
# Non-interactive mode
|
||||
uvx agent-sanitizer \
|
||||
--input ~/.claude/projects \
|
||||
--output ./clean-dataset \
|
||||
--no-interactive
|
||||
```
|
||||
|
||||
### Upload to HuggingFace
|
||||
|
||||
```bash
|
||||
# Login first
|
||||
huggingface-cli login
|
||||
|
||||
# Sanitize and upload in one command
|
||||
uvx agent-sanitizer \
|
||||
--input ~/.claude/projects \
|
||||
--upload myusername/my-coding-dataset
|
||||
|
||||
# Make it private
|
||||
uvx agent-sanitizer \
|
||||
--input ~/.claude/projects \
|
||||
--upload myusername/my-dataset \
|
||||
--private
|
||||
```
|
||||
|
||||
### Use Sanitized Dataset
|
||||
|
||||
After sanitization, use the dataset:
|
||||
|
||||
```python
|
||||
from datasets import load_dataset
|
||||
|
||||
# If uploaded to HuggingFace
|
||||
dataset = load_dataset("username/my-coding-dataset")
|
||||
|
||||
# Or load from local directory
|
||||
import json
|
||||
|
||||
data = []
|
||||
with open("clean-dataset/splits/train.jsonl") as f:
|
||||
for line in f:
|
||||
data.append(json.loads(line))
|
||||
|
||||
print(f"Loaded {len(data)} examples")
|
||||
```
|
||||
|
||||
## Output Format
|
||||
|
||||
Creates a structured dataset:
|
||||
|
||||
```
|
||||
clean-dataset/
|
||||
├── splits/
|
||||
│ ├── train.jsonl
|
||||
│ ├── val.jsonl
|
||||
│ └── test.jsonl
|
||||
├── audit_report.json
|
||||
└── sanitization_summary.json
|
||||
```
|
||||
|
||||
Each JSONL line contains:
|
||||
- `timestamp`: When the interaction occurred
|
||||
- `model`: Which AI model was used
|
||||
- `tokens`: Token usage breakdown
|
||||
- `content`: Array of content blocks (thinking, tool use, text)
|
||||
- `cwd`: Working directory (anonymized)
|
||||
- `git_branch`: Git context
|
||||
|
||||
## Security
|
||||
|
||||
### What Gets Removed
|
||||
|
||||
1. **Credentials**
|
||||
- API keys (OpenAI, Anthropic, etc.)
|
||||
- Passwords and auth tokens
|
||||
- GitHub personal access tokens
|
||||
- SSH/PGP private keys
|
||||
|
||||
2. **Personal Information**
|
||||
- Email addresses (except safe ones like user@example.com)
|
||||
- Phone numbers
|
||||
- SSNs
|
||||
- Personal names
|
||||
|
||||
3. **Cryptocurrency**
|
||||
- Private keys (Ethereum, Bitcoin, etc.)
|
||||
- Seed phrases
|
||||
- Wallet addresses (when not test data)
|
||||
|
||||
### What Gets Preserved
|
||||
|
||||
1. **Test Data**
|
||||
- Hardhat/Ganache test accounts
|
||||
- BIP-39 test seed phrases
|
||||
- Localhost configurations
|
||||
|
||||
2. **Examples**
|
||||
- Code snippets with placeholders
|
||||
- Documentation
|
||||
- Tutorial content
|
||||
|
||||
3. **Technical Context**
|
||||
- Usernames (demonstrates workflows)
|
||||
- Tool usage patterns
|
||||
- Error handling flows
|
||||
|
||||
## Contributing
|
||||
|
||||
Contributions welcome! Please:
|
||||
|
||||
1. Check existing issues
|
||||
2. Create feature branch
|
||||
3. Add tests for new patterns
|
||||
4. Submit pull request
|
||||
|
||||
## Community Datasets
|
||||
|
||||
Share your sanitized dataset:
|
||||
|
||||
1. Upload to HuggingFace with `--upload`
|
||||
2. Tag with `agent-coding-dataset`
|
||||
3. Link to original tool in dataset card
|
||||
|
||||
Example datasets:
|
||||
- [zenlm/agent-coding-dataset](https://huggingface.co/datasets/zenlm/agent-coding-dataset) - 161k Claude Code interactions
|
||||
|
||||
## License
|
||||
|
||||
BSD-3-Clause - See [LICENSE](LICENSE)
|
||||
|
||||
## Links
|
||||
|
||||
- **GitHub**: https://github.com/zenlm/agent-sanitizer
|
||||
- **PyPI**: https://pypi.org/project/agent-sanitizer/
|
||||
- **Example Dataset**: https://huggingface.co/datasets/zenlm/agent-coding-dataset
|
||||
- **Issues**: https://github.com/zenlm/agent-sanitizer/issues
|
||||
|
||||
## Acknowledgments
|
||||
|
||||
Built by [Zen AI](https://zenlm.org) to help the community share coding datasets safely.
|
||||
|
||||
---
|
||||
|
||||
**Made with ❤️ for the AI coding community**
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Agent Sanitizer - Clean AI agent logs for safe dataset sharing."""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
__author__ = "Zen AI"
|
||||
__license__ = "BSD-3-Clause"
|
||||
|
||||
from .sanitizer import AgentSanitizer
|
||||
from .detectors import SecurityDetector
|
||||
|
||||
__all__ = ["AgentSanitizer", "SecurityDetector"]
|
||||
@@ -0,0 +1,302 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Agent Sanitizer CLI - Interactive tool for sanitizing AI agent logs.
|
||||
|
||||
Usage:
|
||||
uvx agent-sanitizer
|
||||
uvx agent-sanitizer --input ~/.claude/projects --output ./clean-dataset
|
||||
uvx agent-sanitizer --upload zenlm/my-dataset
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import click
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.progress import Progress
|
||||
from rich.table import Table
|
||||
from rich.prompt import Confirm, Prompt
|
||||
|
||||
from .sanitizer import AgentSanitizer
|
||||
from .detectors import SecurityDetector
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option(
|
||||
'--input', '-i',
|
||||
type=click.Path(exists=True, path_type=Path),
|
||||
help='Input directory containing agent logs (e.g., ~/.claude/projects)'
|
||||
)
|
||||
@click.option(
|
||||
'--output', '-o',
|
||||
type=click.Path(path_type=Path),
|
||||
help='Output directory for sanitized dataset'
|
||||
)
|
||||
@click.option(
|
||||
'--format',
|
||||
type=click.Choice(['claude', 'openai', 'auto'], case_sensitive=False),
|
||||
default='auto',
|
||||
help='Agent log format (auto-detect by default)'
|
||||
)
|
||||
@click.option(
|
||||
'--dry-run',
|
||||
is_flag=True,
|
||||
help='Show what would be changed without modifying files'
|
||||
)
|
||||
@click.option(
|
||||
'--upload',
|
||||
help='Upload to HuggingFace Hub (e.g., username/dataset-name)'
|
||||
)
|
||||
@click.option(
|
||||
'--private',
|
||||
is_flag=True,
|
||||
help='Make HuggingFace dataset private'
|
||||
)
|
||||
@click.option(
|
||||
'--interactive/--no-interactive',
|
||||
default=True,
|
||||
help='Interactive mode with confirmations'
|
||||
)
|
||||
def main(
|
||||
input: Optional[Path],
|
||||
output: Optional[Path],
|
||||
format: str,
|
||||
dry_run: bool,
|
||||
upload: Optional[str],
|
||||
private: bool,
|
||||
interactive: bool
|
||||
):
|
||||
"""🧹 Sanitize AI agent logs for safe dataset sharing.
|
||||
|
||||
This tool helps you:
|
||||
- Remove PII (names, emails, phone numbers)
|
||||
- Clean credentials (API keys, passwords, tokens)
|
||||
- Anonymize file paths and project names
|
||||
- Upload sanitized data to HuggingFace
|
||||
|
||||
Quick start:
|
||||
uvx agent-sanitizer
|
||||
"""
|
||||
console.print(Panel.fit(
|
||||
"[bold cyan]Agent Sanitizer v0.1.0[/bold cyan]\n"
|
||||
"Clean AI agent logs for safe sharing",
|
||||
border_style="cyan"
|
||||
))
|
||||
|
||||
# Interactive setup if needed
|
||||
if interactive and not input:
|
||||
console.print("\n[yellow]Let's set up your sanitization![/yellow]\n")
|
||||
|
||||
# Detect common agent directories
|
||||
common_paths = detect_agent_directories()
|
||||
if common_paths:
|
||||
console.print("[green]Found agent directories:[/green]")
|
||||
for i, path in enumerate(common_paths, 1):
|
||||
console.print(f" {i}. {path}")
|
||||
|
||||
choice = Prompt.ask(
|
||||
"\nSelect directory or enter custom path",
|
||||
default="1"
|
||||
)
|
||||
|
||||
try:
|
||||
idx = int(choice) - 1
|
||||
if 0 <= idx < len(common_paths):
|
||||
input = common_paths[idx]
|
||||
except ValueError:
|
||||
input = Path(choice).expanduser()
|
||||
else:
|
||||
input = Path(Prompt.ask(
|
||||
"Enter path to agent logs",
|
||||
default="~/.claude/projects"
|
||||
)).expanduser()
|
||||
|
||||
if not input:
|
||||
console.print("[red]Error: No input directory specified[/red]")
|
||||
console.print("Use --input or run interactively")
|
||||
sys.exit(1)
|
||||
|
||||
input = Path(input).expanduser().resolve()
|
||||
|
||||
if not output:
|
||||
output = Path.cwd() / "sanitized-dataset"
|
||||
if interactive:
|
||||
custom = Prompt.ask(
|
||||
f"Output directory",
|
||||
default=str(output)
|
||||
)
|
||||
output = Path(custom)
|
||||
|
||||
output = Path(output).expanduser().resolve()
|
||||
|
||||
# Run audit first
|
||||
console.print(f"\n[cyan]📊 Analyzing logs in {input}...[/cyan]\n")
|
||||
|
||||
detector = SecurityDetector()
|
||||
sanitizer = AgentSanitizer(detector)
|
||||
|
||||
# Scan for issues
|
||||
with Progress() as progress:
|
||||
task = progress.add_task("[cyan]Scanning...", total=None)
|
||||
issues = sanitizer.scan_directory(input, format=format)
|
||||
progress.update(task, completed=True)
|
||||
|
||||
# Show audit results
|
||||
show_audit_results(issues)
|
||||
|
||||
# Confirm if interactive
|
||||
if interactive and issues['total'] > 0:
|
||||
if not Confirm.ask("\n[yellow]Proceed with sanitization?[/yellow]"):
|
||||
console.print("[yellow]Cancelled[/yellow]")
|
||||
return
|
||||
|
||||
# Sanitize
|
||||
if dry_run:
|
||||
console.print("\n[yellow]🔍 DRY RUN - No files will be modified[/yellow]")
|
||||
|
||||
console.print(f"\n[cyan]🧹 Sanitizing dataset...[/cyan]")
|
||||
|
||||
result = sanitizer.sanitize_directory(
|
||||
input_dir=input,
|
||||
output_dir=output,
|
||||
dry_run=dry_run,
|
||||
format=format
|
||||
)
|
||||
|
||||
# Show results
|
||||
show_sanitization_results(result, dry_run)
|
||||
|
||||
# Upload to HuggingFace if requested
|
||||
if upload and not dry_run:
|
||||
if interactive:
|
||||
if not Confirm.ask(f"\n[yellow]Upload to HuggingFace ({upload})?[/yellow]"):
|
||||
console.print("[green]✅ Sanitization complete![/green]")
|
||||
console.print(f"[dim]Dataset saved to: {output}[/dim]")
|
||||
return
|
||||
|
||||
upload_to_huggingface(output, upload, private)
|
||||
|
||||
console.print("\n[green]✅ Done![/green]")
|
||||
if not dry_run:
|
||||
console.print(f"[dim]Dataset saved to: {output}[/dim]")
|
||||
|
||||
|
||||
def detect_agent_directories() -> list[Path]:
|
||||
"""Auto-detect common agent log directories."""
|
||||
common = [
|
||||
Path.home() / ".claude" / "projects",
|
||||
Path.home() / ".cursor" / "logs",
|
||||
Path.home() / ".continue" / "sessions",
|
||||
Path.home() / ".aider" / "history",
|
||||
]
|
||||
return [p for p in common if p.exists()]
|
||||
|
||||
|
||||
def show_audit_results(issues: dict):
|
||||
"""Display audit results in a nice table."""
|
||||
table = Table(title="Security Audit Results")
|
||||
table.add_column("Category", style="cyan")
|
||||
table.add_column("Found", justify="right", style="yellow")
|
||||
table.add_column("Severity", style="magenta")
|
||||
|
||||
severity_map = {
|
||||
'credentials': '🔴 Critical',
|
||||
'names': '🟡 Medium',
|
||||
'emails': '🟡 Medium',
|
||||
'phone_numbers': '🟡 Medium',
|
||||
'crypto_keys': '🔴 Critical',
|
||||
'api_keys': '🟠 High',
|
||||
}
|
||||
|
||||
total = 0
|
||||
for category, count in issues.items():
|
||||
if category == 'total':
|
||||
continue
|
||||
if count > 0:
|
||||
severity = severity_map.get(category, '⚪ Low')
|
||||
table.add_row(category.replace('_', ' ').title(), str(count), severity)
|
||||
total += count
|
||||
|
||||
if total > 0:
|
||||
table.add_section()
|
||||
table.add_row("[bold]Total Issues[/bold]", f"[bold]{total}[/bold]", "")
|
||||
|
||||
console.print(table)
|
||||
|
||||
if total == 0:
|
||||
console.print("[green]✅ No security issues found![/green]")
|
||||
|
||||
|
||||
def show_sanitization_results(result: dict, dry_run: bool):
|
||||
"""Display sanitization results."""
|
||||
console.print()
|
||||
|
||||
table = Table(title="Sanitization Results")
|
||||
table.add_column("Metric", style="cyan")
|
||||
table.add_column("Value", justify="right", style="green")
|
||||
|
||||
table.add_row("Files Processed", f"{result['files_processed']:,}")
|
||||
table.add_row("Lines Processed", f"{result['lines_processed']:,}")
|
||||
table.add_row("Lines Modified", f"{result['lines_modified']:,}")
|
||||
|
||||
if result.get('replacements'):
|
||||
table.add_section()
|
||||
for category, count in result['replacements'].items():
|
||||
if count > 0:
|
||||
table.add_row(
|
||||
f" {category.replace('_', ' ').title()}",
|
||||
str(count)
|
||||
)
|
||||
|
||||
console.print(table)
|
||||
|
||||
|
||||
def upload_to_huggingface(dataset_dir: Path, repo_id: str, private: bool):
|
||||
"""Upload sanitized dataset to HuggingFace."""
|
||||
console.print(f"\n[cyan]📤 Uploading to HuggingFace: {repo_id}...[/cyan]")
|
||||
|
||||
try:
|
||||
from datasets import Dataset, DatasetDict
|
||||
from huggingface_hub import HfApi
|
||||
|
||||
# Load splits
|
||||
splits_dir = dataset_dir / "splits"
|
||||
if not splits_dir.exists():
|
||||
console.print("[red]Error: No splits directory found[/red]")
|
||||
return
|
||||
|
||||
splits = {}
|
||||
for split_file in splits_dir.glob("*.jsonl"):
|
||||
split_name = split_file.stem
|
||||
data = []
|
||||
with open(split_file) as f:
|
||||
for line in f:
|
||||
data.append(json.loads(line))
|
||||
splits[split_name] = Dataset.from_list(data)
|
||||
console.print(f" Loaded {split_name}: {len(data):,} examples")
|
||||
|
||||
if not splits:
|
||||
console.print("[red]Error: No data found[/red]")
|
||||
return
|
||||
|
||||
dataset = DatasetDict(splits)
|
||||
|
||||
console.print(f"\n[yellow]Uploading...[/yellow]")
|
||||
dataset.push_to_hub(repo_id, private=private)
|
||||
|
||||
console.print(f"[green]✅ Uploaded to https://huggingface.co/datasets/{repo_id}[/green]")
|
||||
|
||||
except ImportError:
|
||||
console.print("[red]Error: huggingface_hub not installed[/red]")
|
||||
console.print("Run: pip install huggingface_hub datasets")
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error uploading: {e}[/red]")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Security detectors for finding PII and credentials in agent logs."""
|
||||
|
||||
import re
|
||||
from typing import Dict, List, Set
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class Finding:
|
||||
"""A security finding."""
|
||||
category: str
|
||||
pattern_name: str
|
||||
value: str
|
||||
location: str
|
||||
severity: str # critical, high, medium, low
|
||||
|
||||
|
||||
class SecurityDetector:
|
||||
"""Detect security issues in text."""
|
||||
|
||||
def __init__(self):
|
||||
self.findings: List[Finding] = []
|
||||
|
||||
# Security patterns
|
||||
self.patterns = {
|
||||
# Credentials (CRITICAL)
|
||||
'api_key_openai': (re.compile(r'\bsk-[a-zA-Z0-9]{48,}\b'), 'critical'),
|
||||
'api_key_anthropic': (re.compile(r'\bsk-ant-[a-zA-Z0-9\-]{95,}\b'), 'critical'),
|
||||
'api_key_generic': (re.compile(r'(?:api[_-]?key|apikey)[\s:=]+["\']?([a-zA-Z0-9\-_]{20,})', re.I), 'high'),
|
||||
'password': (re.compile(r'(?:password|passwd|pwd)[\s:=]+["\']([^"\']{6,})["\']', re.I), 'high'),
|
||||
'bearer_token': (re.compile(r'Bearer\s+[a-zA-Z0-9\-._~+/]+=*', re.I), 'high'),
|
||||
'jwt_token': (re.compile(r'eyJ[a-zA-Z0-9_-]+\.eyJ[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+'), 'high'),
|
||||
|
||||
# GitHub tokens
|
||||
'github_pat': (re.compile(r'\bghp_[a-zA-Z0-9]{36}\b'), 'critical'),
|
||||
'github_oauth': (re.compile(r'\bgho_[a-zA-Z0-9]{36}\b'), 'critical'),
|
||||
|
||||
# Cloud providers
|
||||
'aws_key': (re.compile(r'\bAKIA[0-9A-Z]{16}\b'), 'critical'),
|
||||
|
||||
# SSH/PGP keys
|
||||
'ssh_private': (re.compile(r'-----BEGIN (?:RSA|OPENSSH|DSA|EC) PRIVATE KEY-----'), 'critical'),
|
||||
'pgp_private': (re.compile(r'-----BEGIN PGP PRIVATE KEY BLOCK-----'), 'critical'),
|
||||
|
||||
# PII
|
||||
'email': (re.compile(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b'), 'medium'),
|
||||
'phone_us': (re.compile(r'\b(?:\+?1[-.]?)?\(?([0-9]{3})\)?[-.]?([0-9]{3})[-.]?([0-9]{4})\b'), 'medium'),
|
||||
'ssn': (re.compile(r'\b[0-9]{3}-[0-9]{2}-[0-9]{4}\b'), 'critical'),
|
||||
|
||||
# Crypto
|
||||
'eth_private_key': (re.compile(r'\b0x[a-fA-F0-9]{64}\b'), 'critical'),
|
||||
'eth_address': (re.compile(r'\b0x[a-fA-F0-9]{40}\b'), 'low'),
|
||||
}
|
||||
|
||||
# Safe patterns (reduce false positives)
|
||||
self.safe_patterns = {
|
||||
# Safe emails
|
||||
'user@example.com', 'test@example.com', 'admin@example.com',
|
||||
'email@example.com', 'noreply@example.com',
|
||||
# Safe addresses
|
||||
'0x0000000000000000000000000000000000000000',
|
||||
'0xffffffffffffffffffffffffffffffffffffffff',
|
||||
'0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef',
|
||||
# Hardhat test account
|
||||
'0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266',
|
||||
'0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80',
|
||||
}
|
||||
|
||||
def is_safe(self, text: str) -> bool:
|
||||
"""Check if text is a known safe pattern."""
|
||||
text_lower = text.lower()
|
||||
return any(safe.lower() in text_lower for safe in self.safe_patterns)
|
||||
|
||||
def scan_text(self, text: str, location: str = "") -> Dict[str, int]:
|
||||
"""Scan text for security issues."""
|
||||
if not text or not isinstance(text, str):
|
||||
return {}
|
||||
|
||||
issues = {}
|
||||
|
||||
for pattern_name, (pattern, severity) in self.patterns.items():
|
||||
matches = pattern.findall(text)
|
||||
if matches:
|
||||
# Filter safe patterns
|
||||
if pattern_name in ['email', 'eth_address', 'eth_private_key']:
|
||||
matches = [m for m in matches if not self.is_safe(str(m))]
|
||||
|
||||
if matches:
|
||||
category = self._get_category(pattern_name)
|
||||
issues[category] = issues.get(category, 0) + len(matches)
|
||||
|
||||
# Store findings
|
||||
for match in matches[:3]: # Limit to first 3
|
||||
self.findings.append(Finding(
|
||||
category=category,
|
||||
pattern_name=pattern_name,
|
||||
value=str(match)[:50], # Truncate long values
|
||||
location=location,
|
||||
severity=severity
|
||||
))
|
||||
|
||||
return issues
|
||||
|
||||
def _get_category(self, pattern_name: str) -> str:
|
||||
"""Map pattern name to category."""
|
||||
if 'key' in pattern_name or 'token' in pattern_name:
|
||||
return 'credentials'
|
||||
elif 'password' in pattern_name or 'jwt' in pattern_name:
|
||||
return 'credentials'
|
||||
elif 'email' in pattern_name:
|
||||
return 'emails'
|
||||
elif 'phone' in pattern_name:
|
||||
return 'phone_numbers'
|
||||
elif 'ssh' in pattern_name or 'pgp' in pattern_name:
|
||||
return 'crypto_keys'
|
||||
elif 'eth' in pattern_name or 'btc' in pattern_name:
|
||||
return 'crypto_keys'
|
||||
elif 'ssn' in pattern_name:
|
||||
return 'pii'
|
||||
else:
|
||||
return 'other'
|
||||
|
||||
def get_summary(self) -> Dict[str, int]:
|
||||
"""Get summary of findings by category."""
|
||||
summary = {}
|
||||
for finding in self.findings:
|
||||
summary[finding.category] = summary.get(finding.category, 0) + 1
|
||||
summary['total'] = len(self.findings)
|
||||
return summary
|
||||
@@ -0,0 +1,259 @@
|
||||
"""Core sanitizer logic for cleaning agent logs."""
|
||||
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Dict, Optional
|
||||
from tqdm import tqdm
|
||||
|
||||
from .detectors import SecurityDetector
|
||||
|
||||
|
||||
class AgentSanitizer:
|
||||
"""Sanitize agent logs for safe sharing."""
|
||||
|
||||
def __init__(self, detector: Optional[SecurityDetector] = None):
|
||||
self.detector = detector or SecurityDetector()
|
||||
self.stats = {
|
||||
'files_processed': 0,
|
||||
'lines_processed': 0,
|
||||
'lines_modified': 0,
|
||||
'replacements': {},
|
||||
}
|
||||
|
||||
# Replacement patterns
|
||||
self.replacements = [
|
||||
# Credentials
|
||||
(re.compile(r'\bsk-[a-zA-Z0-9]{48,}\b'), '[REDACTED_API_KEY]'),
|
||||
(re.compile(r'\bsk-ant-[a-zA-Z0-9\-]{95,}\b'), '[REDACTED_API_KEY]'),
|
||||
(re.compile(r'\bghp_[a-zA-Z0-9]{36}\b'), '[REDACTED_GITHUB_TOKEN]'),
|
||||
(re.compile(r'Bearer\s+[a-zA-Z0-9\-._~+/]+=*', re.I), 'Bearer [REDACTED_TOKEN]'),
|
||||
|
||||
# Passwords (conservative - only if quoted)
|
||||
(re.compile(r'(?:password|passwd|pwd)[\s:=]+["\']([^"\']{6,})["\']', re.I),
|
||||
lambda m: m.group(0).replace(m.group(1), '[REDACTED_PASSWORD]')),
|
||||
|
||||
# SSH/PGP keys
|
||||
(re.compile(r'-----BEGIN .*? PRIVATE KEY-----.*?-----END .*? PRIVATE KEY-----', re.S),
|
||||
'[REDACTED_PRIVATE_KEY]'),
|
||||
]
|
||||
|
||||
# Email safe list
|
||||
self.safe_emails = {'user@example.com', 'test@example.com', 'admin@example.com'}
|
||||
|
||||
def scan_directory(self, directory: Path, format: str = 'auto') -> Dict[str, int]:
|
||||
"""Scan directory and return issue summary."""
|
||||
issues = {}
|
||||
|
||||
# Find log files based on format
|
||||
if format == 'auto':
|
||||
format = self._detect_format(directory)
|
||||
|
||||
files = self._find_log_files(directory, format)
|
||||
|
||||
for file_path in tqdm(files, desc="Scanning"):
|
||||
file_issues = self._scan_file(file_path, format)
|
||||
for category, count in file_issues.items():
|
||||
issues[category] = issues.get(category, 0) + count
|
||||
|
||||
issues['total'] = sum(v for k, v in issues.items() if k != 'total')
|
||||
return issues
|
||||
|
||||
def sanitize_directory(
|
||||
self,
|
||||
input_dir: Path,
|
||||
output_dir: Path,
|
||||
dry_run: bool = False,
|
||||
format: str = 'auto'
|
||||
) -> Dict:
|
||||
"""Sanitize all files in directory."""
|
||||
if format == 'auto':
|
||||
format = self._detect_format(input_dir)
|
||||
|
||||
files = self._find_log_files(input_dir, format)
|
||||
|
||||
if not dry_run:
|
||||
# Create backup
|
||||
backup_dir = input_dir.parent / f"{input_dir.name}.backup.{datetime.now().strftime('%Y%m%d_%H%M%S')}"
|
||||
|
||||
# Create output structure
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(output_dir / 'splits').mkdir(exist_ok=True)
|
||||
|
||||
for file_path in tqdm(files, desc="Sanitizing"):
|
||||
relative = file_path.relative_to(input_dir)
|
||||
output_file = output_dir / relative
|
||||
|
||||
self._sanitize_file(file_path, output_file, format, dry_run)
|
||||
|
||||
return self.stats
|
||||
|
||||
def _detect_format(self, directory: Path) -> str:
|
||||
"""Auto-detect log format."""
|
||||
# Check for Claude Code logs
|
||||
if (directory / 'projects').exists() or directory.name == 'projects':
|
||||
return 'claude'
|
||||
# Check for Cursor logs
|
||||
if 'cursor' in str(directory).lower():
|
||||
return 'cursor'
|
||||
# Check for Continue logs
|
||||
if 'continue' in str(directory).lower():
|
||||
return 'continue'
|
||||
# Default
|
||||
return 'claude'
|
||||
|
||||
def _find_log_files(self, directory: Path, format: str) -> list[Path]:
|
||||
"""Find log files based on format."""
|
||||
if format == 'claude':
|
||||
# Claude Code: Look for JSONL files in projects directory
|
||||
if directory.name == 'projects':
|
||||
return list(directory.glob('*/*.jsonl'))
|
||||
elif (directory / 'projects').exists():
|
||||
return list((directory / 'projects').glob('*/*.jsonl'))
|
||||
else:
|
||||
return list(directory.rglob('*.jsonl'))
|
||||
else:
|
||||
# Generic: Find all JSONL files
|
||||
return list(directory.rglob('*.jsonl'))
|
||||
|
||||
def _scan_file(self, file_path: Path, format: str) -> Dict[str, int]:
|
||||
"""Scan single file for issues."""
|
||||
issues = {}
|
||||
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
|
||||
for line_num, line in enumerate(f, 1):
|
||||
try:
|
||||
data = json.loads(line)
|
||||
text = self._extract_text(data, format)
|
||||
location = f"{file_path.name}:L{line_num}"
|
||||
file_issues = self.detector.scan_text(text, location)
|
||||
|
||||
for category, count in file_issues.items():
|
||||
issues[category] = issues.get(category, 0) + count
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return issues
|
||||
|
||||
def _sanitize_file(
|
||||
self,
|
||||
input_path: Path,
|
||||
output_path: Path,
|
||||
format: str,
|
||||
dry_run: bool
|
||||
):
|
||||
"""Sanitize single file."""
|
||||
cleaned_lines = []
|
||||
lines_modified = 0
|
||||
|
||||
try:
|
||||
with open(input_path, 'r', encoding='utf-8', errors='replace') as f:
|
||||
for line in f:
|
||||
self.stats['lines_processed'] += 1
|
||||
|
||||
try:
|
||||
data = json.loads(line)
|
||||
cleaned_data, was_modified = self._sanitize_interaction(data, format)
|
||||
|
||||
if was_modified:
|
||||
lines_modified += 1
|
||||
self.stats['lines_modified'] += 1
|
||||
|
||||
cleaned_lines.append(json.dumps(cleaned_data, ensure_ascii=False))
|
||||
except json.JSONDecodeError:
|
||||
cleaned_lines.append(line.rstrip())
|
||||
|
||||
if not dry_run:
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
for line in cleaned_lines:
|
||||
f.write(line + '\n')
|
||||
|
||||
self.stats['files_processed'] += 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error processing {input_path}: {e}")
|
||||
|
||||
def _extract_text(self, data: dict, format: str) -> str:
|
||||
"""Extract all text from interaction."""
|
||||
texts = []
|
||||
|
||||
if format == 'claude':
|
||||
for item in data.get('content', []):
|
||||
if item.get('type') == 'thinking':
|
||||
texts.append(item.get('thinking', ''))
|
||||
elif item.get('type') == 'text':
|
||||
texts.append(item.get('text', ''))
|
||||
texts.append(data.get('cwd', ''))
|
||||
else:
|
||||
# Generic extraction
|
||||
texts.append(json.dumps(data))
|
||||
|
||||
return ' '.join(texts)
|
||||
|
||||
def _sanitize_interaction(self, data: dict, format: str) -> tuple[dict, bool]:
|
||||
"""Sanitize single interaction."""
|
||||
modified = False
|
||||
|
||||
if format == 'claude':
|
||||
# Sanitize content blocks
|
||||
for item in data.get('content', []):
|
||||
if item.get('type') == 'thinking' and 'thinking' in item:
|
||||
cleaned, was_modified = self._clean_text(item['thinking'])
|
||||
if was_modified:
|
||||
item['thinking'] = cleaned
|
||||
modified = True
|
||||
|
||||
if item.get('type') == 'text' and 'text' in item:
|
||||
cleaned, was_modified = self._clean_text(item['text'])
|
||||
if was_modified:
|
||||
item['text'] = cleaned
|
||||
modified = True
|
||||
|
||||
# Sanitize cwd
|
||||
if 'cwd' in data:
|
||||
cleaned, was_modified = self._clean_text(data['cwd'])
|
||||
if was_modified:
|
||||
data['cwd'] = cleaned
|
||||
modified = True
|
||||
else:
|
||||
# Generic sanitization
|
||||
data_str = json.dumps(data)
|
||||
cleaned, was_modified = self._clean_text(data_str)
|
||||
if was_modified:
|
||||
data = json.loads(cleaned)
|
||||
modified = True
|
||||
|
||||
return data, modified
|
||||
|
||||
def _clean_text(self, text: str) -> tuple[str, bool]:
|
||||
"""Clean text and return (cleaned_text, was_modified)."""
|
||||
if not text or not isinstance(text, str):
|
||||
return text, False
|
||||
|
||||
original = text
|
||||
|
||||
for pattern, replacement in self.replacements:
|
||||
if callable(replacement):
|
||||
text = pattern.sub(replacement, text)
|
||||
else:
|
||||
text = pattern.sub(replacement, text)
|
||||
|
||||
# Clean emails (keep safe ones)
|
||||
email_pattern = re.compile(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b')
|
||||
emails = email_pattern.findall(text)
|
||||
for email in emails:
|
||||
if email.lower() not in self.safe_emails and not email.endswith('@example.com'):
|
||||
text = text.replace(email, 'user@example.com')
|
||||
self.stats['replacements']['emails'] = self.stats['replacements'].get('emails', 0) + 1
|
||||
|
||||
# Track replacements
|
||||
if text != original:
|
||||
self.stats['replacements']['credentials'] = self.stats['replacements'].get('credentials', 0) + 1
|
||||
|
||||
return text, text != original
|
||||
@@ -0,0 +1,62 @@
|
||||
[project]
|
||||
name = "agent-sanitizer"
|
||||
version = "0.1.0"
|
||||
description = "Sanitize AI agent logs for safe dataset sharing"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.8"
|
||||
license = {text = "BSD-3-Clause"}
|
||||
authors = [
|
||||
{name = "Zen AI", email = "hello@zenlm.org"}
|
||||
]
|
||||
keywords = ["ai", "agents", "dataset", "privacy", "sanitization", "claude", "openai"]
|
||||
classifiers = [
|
||||
"Development Status :: 4 - Beta",
|
||||
"Intended Audience :: Developers",
|
||||
"Intended Audience :: Science/Research",
|
||||
"License :: OSI Approved :: BSD License",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
||||
]
|
||||
|
||||
dependencies = [
|
||||
"rich>=13.0.0",
|
||||
"click>=8.0.0",
|
||||
"tqdm>=4.65.0",
|
||||
"huggingface-hub>=0.20.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=7.0.0",
|
||||
"black>=23.0.0",
|
||||
"ruff>=0.1.0",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/zenlm/agent-sanitizer"
|
||||
Documentation = "https://github.com/zenlm/agent-sanitizer#readme"
|
||||
Repository = "https://github.com/zenlm/agent-sanitizer"
|
||||
Issues = "https://github.com/zenlm/agent-sanitizer/issues"
|
||||
|
||||
[project.scripts]
|
||||
agent-sanitizer = "agent_sanitizer.cli:main"
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["agent_sanitizer"]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
target-version = "py38"
|
||||
|
||||
[tool.black]
|
||||
line-length = 100
|
||||
target-version = ["py38", "py39", "py310", "py311", "py312"]
|
||||
Reference in New Issue
Block a user