CVE Utils

简体中文

English

简体中文

English

Appearance

漏洞库管理 ​

本示例展示如何使用 CVE Utils 管理大型漏洞数据库，包括数据导入、清洗、验证、分类和导出等完整的数据管理流程。

场景描述 ​

作为安全产品开发者或漏洞数据库维护者，您需要：

  • 从多个数据源导入 CVE 数据
  • 清洗和验证数据质量
  • 按各种条件分类和组织数据
  • 提供高效的查询和导出功能
  • 维护数据的一致性和完整性

完整示例 ​

1. 漏洞数据库管理器 ​

go
package main

import (
    "encoding/json"
    "fmt"
    "os"
    "sort"
    "strings"
    "time"
    "github.com/scagogogo/cve"
)

// VulnerabilityRecord 漏洞记录结构
type VulnerabilityRecord struct {
    CVE         string    `json:"cve"`
    Year        int       `json:"year"`
    Sequence    int       `json:"sequence"`
    Title       string    `json:"title,omitempty"`
    Description string    `json:"description,omitempty"`
    Severity    string    `json:"severity,omitempty"`
    Source      string    `json:"source,omitempty"`
    ImportDate  time.Time `json:"import_date"`
    LastUpdate  time.Time `json:"last_update"`
}

// VulnerabilityDatabase 漏洞数据库管理器
type VulnerabilityDatabase struct {
    records    map[string]*VulnerabilityRecord
    sources    map[string]int // 数据源统计
    statistics DatabaseStats
}

// DatabaseStats 数据库统计信息
type DatabaseStats struct {
    TotalRecords    int
    ValidRecords    int
    InvalidRecords  int
    YearDistribution map[string]int
    SourceDistribution map[string]int
    LastUpdate      time.Time
}

func NewVulnerabilityDatabase() *VulnerabilityDatabase {
    return &VulnerabilityDatabase{
        records: make(map[string]*VulnerabilityRecord),
        sources: make(map[string]int),
        statistics: DatabaseStats{
            YearDistribution:   make(map[string]int),
            SourceDistribution: make(map[string]int),
        },
    }
}

// ImportFromCSV 从 CSV 文件导入数据
func (vdb *VulnerabilityDatabase) ImportFromCSV(filename string) error {
    fmt.Printf("从 CSV 文件导入数据: %s\n", filename)
    
    file, err := os.Open(filename)
    if err != nil {
        return fmt.Errorf("无法打开文件: %v", err)
    }
    defer file.Close()
    
    reader := csv.NewReader(file)
    records, err := reader.ReadAll()
    if err != nil {
        return fmt.Errorf("读取 CSV 失败: %v", err)
    }
    
    imported := 0
    skipped := 0
    
    // 跳过标题行
    for i, record := range records[1:] {
        if len(record) < 2 {
            skipped++
            continue
        }
        
        cveId := strings.TrimSpace(record[0])
        source := "CSV Import"
        if len(record) > 2 {
            source = strings.TrimSpace(record[2])
        }
        
        if vdb.AddRecord(cveId, "", "", "", source) {
            imported++
        } else {
            skipped++
        }
        
        if (i+1)%1000 == 0 {
            fmt.Printf("已处理 %d 条记录...\n", i+1)
        }
    }
    
    fmt.Printf("导入完成: %d 条成功, %d 条跳过\n", imported, skipped)
    return nil
}

// ImportFromText 从文本中提取并导入 CVE
func (vdb *VulnerabilityDatabase) ImportFromText(text, source string) int {
    cves := cve.ExtractCve(text)
    imported := 0
    
    for _, cveId := range cves {
        if vdb.AddRecord(cveId, "", "", "", source) {
            imported++
        }
    }
    
    return imported
}

// AddRecord 添加漏洞记录
func (vdb *VulnerabilityDatabase) AddRecord(cveId, title, description, severity, source string) bool {
    // 验证 CVE 格式
    if !cve.ValidateCve(cveId) {
        return false
    }
    
    // 格式化 CVE
    formattedCve := cve.Format(cveId)
    
    // 检查是否已存在
    if existing, exists := vdb.records[formattedCve]; exists {
        // 更新现有记录
        existing.LastUpdate = time.Now()
        if title != "" {
            existing.Title = title
        }
        if description != "" {
            existing.Description = description
        }
        if severity != "" {
            existing.Severity = severity
        }
        return true
    }
    
    // 创建新记录
    record := &VulnerabilityRecord{
        CVE:         formattedCve,
        Year:        cve.ExtractCveYearAsInt(formattedCve),
        Sequence:    cve.ExtractCveSeqAsInt(formattedCve),
        Title:       title,
        Description: description,
        Severity:    severity,
        Source:      source,
        ImportDate:  time.Now(),
        LastUpdate:  time.Now(),
    }
    
    vdb.records[formattedCve] = record
    vdb.sources[source]++
    
    return true
}

// CleanDatabase 清洗数据库
func (vdb *VulnerabilityDatabase) CleanDatabase() *CleaningReport {
    fmt.Println("开始数据库清洗...")
    
    report := &CleaningReport{
        StartTime: time.Now(),
    }
    
    var toRemove []string
    
    for cveId, record := range vdb.records {
        // 验证 CVE 格式
        if !cve.ValidateCve(cveId) {
            toRemove = append(toRemove, cveId)
            report.InvalidCves = append(report.InvalidCves, cveId)
            continue
        }
        
        // 检查年份合理性
        if !cve.IsCveYearOk(cveId, 10) {
            toRemove = append(toRemove, cveId)
            report.InvalidYears = append(report.InvalidYears, cveId)
            continue
        }
        
        // 验证记录完整性
        if record.Year != cve.ExtractCveYearAsInt(cveId) ||
           record.Sequence != cve.ExtractCveSeqAsInt(cveId) {
            // 修复记录
            record.Year = cve.ExtractCveYearAsInt(cveId)
            record.Sequence = cve.ExtractCveSeqAsInt(cveId)
            record.LastUpdate = time.Now()
            report.FixedRecords = append(report.FixedRecords, cveId)
        }
    }
    
    // 移除无效记录
    for _, cveId := range toRemove {
        delete(vdb.records, cveId)
    }
    
    report.EndTime = time.Now()
    report.RemovedCount = len(toRemove)
    report.TotalProcessed = len(vdb.records) + len(toRemove)
    
    fmt.Printf("清洗完成: 处理 %d 条记录, 移除 %d 条无效记录, 修复 %d 条记录\n",
        report.TotalProcessed, report.RemovedCount, len(report.FixedRecords))
    
    return report
}

// CleaningReport 清洗报告
type CleaningReport struct {
    StartTime       time.Time
    EndTime         time.Time
    TotalProcessed  int
    RemovedCount    int
    InvalidCves     []string
    InvalidYears    []string
    FixedRecords    []string
}

2. 查询和过滤功能 ​

go
// QueryOptions 查询选项
type QueryOptions struct {
    Year        *int
    StartYear   *int
    EndYear     *int
    RecentYears *int
    Source      string
    Severity    string
    Keyword     string
    Limit       int
    Offset      int
}

// Query 查询漏洞记录
func (vdb *VulnerabilityDatabase) Query(options QueryOptions) []*VulnerabilityRecord {
    var results []*VulnerabilityRecord
    
    // 获取所有 CVE ID
    var cveIds []string
    for cveId := range vdb.records {
        cveIds = append(cveIds, cveId)
    }
    
    // 按年份过滤
    if options.Year != nil {
        cveIds = cve.FilterCvesByYear(cveIds, *options.Year)
    } else if options.StartYear != nil && options.EndYear != nil {
        cveIds = cve.FilterCvesByYearRange(cveIds, *options.StartYear, *options.EndYear)
    } else if options.RecentYears != nil {
        cveIds = cve.GetRecentCves(cveIds, *options.RecentYears)
    }
    
    // 应用其他过滤条件
    for _, cveId := range cveIds {
        record := vdb.records[cveId]
        
        // 按数据源过滤
        if options.Source != "" && record.Source != options.Source {
            continue
        }
        
        // 按严重性过滤
        if options.Severity != "" && record.Severity != options.Severity {
            continue
        }
        
        // 按关键词过滤
        if options.Keyword != "" {
            keyword := strings.ToLower(options.Keyword)
            if !strings.Contains(strings.ToLower(record.Title), keyword) &&
               !strings.Contains(strings.ToLower(record.Description), keyword) {
                continue
            }
        }
        
        results = append(results, record)
    }
    
    // 排序
    sort.Slice(results, func(i, j int) bool {
        return cve.CompareCves(results[i].CVE, results[j].CVE) < 0
    })
    
    // 分页
    if options.Offset > 0 && options.Offset < len(results) {
        results = results[options.Offset:]
    }
    
    if options.Limit > 0 && options.Limit < len(results) {
        results = results[:options.Limit]
    }
    
    return results
}

// GetStatistics 获取数据库统计信息
func (vdb *VulnerabilityDatabase) GetStatistics() DatabaseStats {
    stats := DatabaseStats{
        TotalRecords:       len(vdb.records),
        YearDistribution:   make(map[string]int),
        SourceDistribution: make(map[string]int),
        LastUpdate:        time.Now(),
    }
    
    // 统计年份分布
    var cveIds []string
    for cveId := range vdb.records {
        cveIds = append(cveIds, cveId)
    }
    
    grouped := cve.GroupByYear(cveIds)
    for year, yearCves := range grouped {
        stats.YearDistribution[year] = len(yearCves)
    }
    
    // 统计数据源分布
    for source, count := range vdb.sources {
        stats.SourceDistribution[source] = count
    }
    
    // 统计有效/无效记录
    validCount := 0
    for cveId := range vdb.records {
        if cve.ValidateCve(cveId) {
            validCount++
        }
    }
    
    stats.ValidRecords = validCount
    stats.InvalidRecords = stats.TotalRecords - validCount
    
    vdb.statistics = stats
    return stats
}

// PrintStatistics 打印统计信息
func (vdb *VulnerabilityDatabase) PrintStatistics() {
    stats := vdb.GetStatistics()
    
    fmt.Println("\n" + strings.Repeat("=", 50))
    fmt.Println("           数据库统计信息")
    fmt.Println(strings.Repeat("=", 50))
    
    fmt.Printf("总记录数: %d\n", stats.TotalRecords)
    fmt.Printf("有效记录: %d (%.1f%%)\n", 
        stats.ValidRecords, 
        float64(stats.ValidRecords)/float64(stats.TotalRecords)*100)
    fmt.Printf("无效记录: %d (%.1f%%)\n", 
        stats.InvalidRecords,
        float64(stats.InvalidRecords)/float64(stats.TotalRecords)*100)
    
    // 年份分布
    fmt.Println("\n年份分布 (Top 10):")
    type yearCount struct {
        year  string
        count int
    }
    
    var yearCounts []yearCount
    for year, count := range stats.YearDistribution {
        yearCounts = append(yearCounts, yearCount{year, count})
    }
    
    sort.Slice(yearCounts, func(i, j int) bool {
        return yearCounts[i].count > yearCounts[j].count
    })
    
    for i, yc := range yearCounts {
        if i >= 10 {
            break
        }
        percentage := float64(yc.count) / float64(stats.TotalRecords) * 100
        fmt.Printf("  %s年: %5d个 (%5.1f%%)\n", yc.year, yc.count, percentage)
    }
    
    // 数据源分布
    fmt.Println("\n数据源分布:")
    for source, count := range stats.SourceDistribution {
        percentage := float64(count) / float64(stats.TotalRecords) * 100
        fmt.Printf("  %-20s: %5d个 (%5.1f%%)\n", source, count, percentage)
    }
    
    fmt.Printf("\n最后更新: %s\n", stats.LastUpdate.Format("2006-01-02 15:04:05"))
    fmt.Println(strings.Repeat("=", 50))
}

3. 数据导出功能 ​

go
// ExportToJSON 导出为 JSON 格式
func (vdb *VulnerabilityDatabase) ExportToJSON(filename string, options QueryOptions) error {
    records := vdb.Query(options)
    
    file, err := os.Create(filename)
    if err != nil {
        return err
    }
    defer file.Close()
    
    encoder := json.NewEncoder(file)
    encoder.SetIndent("", "  ")
    
    exportData := map[string]interface{}{
        "export_time":    time.Now(),
        "total_records":  len(records),
        "query_options":  options,
        "records":        records,
    }
    
    return encoder.Encode(exportData)
}

// ExportToCSV 导出为 CSV 格式
func (vdb *VulnerabilityDatabase) ExportToCSV(filename string, options QueryOptions) error {
    records := vdb.Query(options)
    
    file, err := os.Create(filename)
    if err != nil {
        return err
    }
    defer file.Close()
    
    writer := csv.NewWriter(file)
    defer writer.Flush()
    
    // 写入标题行
    headers := []string{
        "CVE", "Year", "Sequence", "Title", "Description", 
        "Severity", "Source", "Import_Date", "Last_Update",
    }
    writer.Write(headers)
    
    // 写入数据行
    for _, record := range records {
        row := []string{
            record.CVE,
            fmt.Sprintf("%d", record.Year),
            fmt.Sprintf("%d", record.Sequence),
            record.Title,
            record.Description,
            record.Severity,
            record.Source,
            record.ImportDate.Format("2006-01-02 15:04:05"),
            record.LastUpdate.Format("2006-01-02 15:04:05"),
        }
        writer.Write(row)
    }
    
    return nil
}

// BackupDatabase 备份数据库
func (vdb *VulnerabilityDatabase) BackupDatabase(backupDir string) error {
    timestamp := time.Now().Format("20060102_150405")
    
    // 创建备份目录
    if err := os.MkdirAll(backupDir, 0755); err != nil {
        return err
    }
    
    // 备份完整数据
    fullBackupFile := filepath.Join(backupDir, fmt.Sprintf("full_backup_%s.json", timestamp))
    if err := vdb.ExportToJSON(fullBackupFile, QueryOptions{}); err != nil {
        return fmt.Errorf("完整备份失败: %v", err)
    }
    
    // 备份统计信息
    statsFile := filepath.Join(backupDir, fmt.Sprintf("statistics_%s.json", timestamp))
    stats := vdb.GetStatistics()
    
    file, err := os.Create(statsFile)
    if err != nil {
        return err
    }
    defer file.Close()
    
    encoder := json.NewEncoder(file)
    encoder.SetIndent("", "  ")
    if err := encoder.Encode(stats); err != nil {
        return fmt.Errorf("统计信息备份失败: %v", err)
    }
    
    fmt.Printf("备份完成:\n")
    fmt.Printf("  完整数据: %s\n", fullBackupFile)
    fmt.Printf("  统计信息: %s\n", statsFile)
    
    return nil
}

4. 主程序示例 ​

go
func main() {
    // 创建数据库管理器
    vdb := NewVulnerabilityDatabase()
    
    fmt.Println("=== CVE 漏洞数据库管理系统 ===")
    
    // 模拟从多个数据源导入数据
    fmt.Println("\n1. 数据导入阶段")
    
    // 从文本导入
    securityBulletins := []string{
        "安全公告：发现严重漏洞 CVE-2021-44228、CVE-2022-12345",
        "更新修复了 CVE-2023-1234 和 cve-2023-5678",
        "紧急补丁：CVE-2024-1111 到 CVE-2024-1115",
    }
    
    for i, bulletin := range securityBulletins {
        source := fmt.Sprintf("Security Bulletin %d", i+1)
        imported := vdb.ImportFromText(bulletin, source)
        fmt.Printf("从 %s 导入 %d 个 CVE\n", source, imported)
    }
    
    // 手动添加一些记录
    testRecords := []struct {
        cve, title, severity string
    }{
        {"CVE-2021-44228", "Apache Log4j RCE", "Critical"},
        {"CVE-2022-12345", "Custom Component Privilege Escalation", "High"},
        {"CVE-2023-1234", "Third-party Library Information Disclosure", "Medium"},
        {"CVE-2024-5678", "Zero-day Vulnerability", "Critical"},
    }
    
    for _, record := range testRecords {
        vdb.AddRecord(record.cve, record.title, "", record.severity, "Manual Entry")
    }
    
    fmt.Printf("手动添加 %d 个详细记录\n", len(testRecords))
    
    // 2. 数据清洗
    fmt.Println("\n2. 数据清洗阶段")
    cleaningReport := vdb.CleanDatabase()
    
    if len(cleaningReport.FixedRecords) > 0 {
        fmt.Printf("修复的记录: %v\n", cleaningReport.FixedRecords)
    }
    
    // 3. 统计分析
    fmt.Println("\n3. 统计分析阶段")
    vdb.PrintStatistics()
    
    // 4. 查询示例
    fmt.Println("\n4. 查询示例")
    
    // 查询最近2年的 CVE
    recentOptions := QueryOptions{RecentYears: &[]int{2}[0]}
    recentCves := vdb.Query(recentOptions)
    fmt.Printf("最近2年的 CVE (%d个): ", len(recentCves))
    for _, record := range recentCves {
        fmt.Printf("%s ", record.CVE)
    }
    fmt.Println()
    
    // 查询高危漏洞
    criticalOptions := QueryOptions{Severity: "Critical"}
    criticalCves := vdb.Query(criticalOptions)
    fmt.Printf("高危漏洞 (%d个): ", len(criticalCves))
    for _, record := range criticalCves {
        fmt.Printf("%s ", record.CVE)
    }
    fmt.Println()
    
    // 5. 数据导出
    fmt.Println("\n5. 数据导出阶段")
    
    // 导出所有数据为 JSON
    if err := vdb.ExportToJSON("all_vulnerabilities.json", QueryOptions{}); err != nil {
        fmt.Printf("JSON 导出失败: %v\n", err)
    } else {
        fmt.Println("已导出所有数据到 all_vulnerabilities.json")
    }
    
    // 导出最近数据为 CSV
    if err := vdb.ExportToCSV("recent_vulnerabilities.csv", recentOptions); err != nil {
        fmt.Printf("CSV 导出失败: %v\n", err)
    } else {
        fmt.Println("已导出最近数据到 recent_vulnerabilities.csv")
    }
    
    // 6. 数据备份
    fmt.Println("\n6. 数据备份阶段")
    if err := vdb.BackupDatabase("./backups"); err != nil {
        fmt.Printf("备份失败: %v\n", err)
    }
    
    fmt.Println("\n=== 管理流程完成 ===")
}

高级功能 ​

1. 数据同步 ​

go
func (vdb *VulnerabilityDatabase) SyncWithRemote(remoteURL string) error {
    // 从远程数据源同步数据
    resp, err := http.Get(remoteURL)
    if err != nil {
        return err
    }
    defer resp.Body.Close()
    
    var remoteData []VulnerabilityRecord
    if err := json.NewDecoder(resp.Body).Decode(&remoteData); err != nil {
        return err
    }
    
    syncCount := 0
    for _, record := range remoteData {
        if vdb.AddRecord(record.CVE, record.Title, record.Description, 
                         record.Severity, "Remote Sync") {
            syncCount++
        }
    }
    
    fmt.Printf("同步完成: %d 个记录\n", syncCount)
    return nil
}

2. 数据验证 ​

go
func (vdb *VulnerabilityDatabase) ValidateDatabase() []ValidationError {
    var errors []ValidationError
    
    for cveId, record := range vdb.records {
        // 验证 CVE 格式
        if !cve.ValidateCve(cveId) {
            errors = append(errors, ValidationError{
                CVE:     cveId,
                Type:    "FORMAT_ERROR",
                Message: "无效的 CVE 格式",
            })
        }
        
        // 验证年份一致性
        if record.Year != cve.ExtractCveYearAsInt(cveId) {
            errors = append(errors, ValidationError{
                CVE:     cveId,
                Type:    "YEAR_MISMATCH",
                Message: "年份字段与 CVE 不匹配",
            })
        }
        
        // 验证序列号一致性
        if record.Sequence != cve.ExtractCveSeqAsInt(cveId) {
            errors = append(errors, ValidationError{
                CVE:     cveId,
                Type:    "SEQUENCE_MISMATCH",
                Message: "序列号字段与 CVE 不匹配",
            })
        }
    }
    
    return errors
}

type ValidationError struct {
    CVE     string
    Type    string
    Message string
}

3. 性能监控 ​

go
func (vdb *VulnerabilityDatabase) GetPerformanceMetrics() PerformanceMetrics {
    start := time.Now()
    
    // 测试查询性能
    queryStart := time.Now()
    vdb.Query(QueryOptions{Limit: 1000})
    queryDuration := time.Since(queryStart)
    
    // 测试统计性能
    statsStart := time.Now()
    vdb.GetStatistics()
    statsDuration := time.Since(statsStart)
    
    return PerformanceMetrics{
        TotalRecords:    len(vdb.records),
        QueryTime:       queryDuration,
        StatisticsTime:  statsDuration,
        MemoryUsage:     getMemoryUsage(),
        LastMeasurement: time.Now(),
    }
}

type PerformanceMetrics struct {
    TotalRecords    int
    QueryTime       time.Duration
    StatisticsTime  time.Duration
    MemoryUsage     uint64
    LastMeasurement time.Time
}

最佳实践 ​

  1. 定期清洗: 设置定期任务清洗数据
  2. 增量导入: 对大数据集使用增量导入
  3. 索引优化: 为常用查询字段建立索引
  4. 备份策略: 实施定期备份和版本控制
  5. 监控告警: 监控数据质量和系统性能

这个示例展示了如何构建一个完整的漏洞数据库管理系统，可以根据实际需求扩展更多功能。

基于 MIT 许可证发布。

Copyright © 2024 scagogogo