Class: SwarmMemory::Optimization::Defragmenter

Inherits:
Object
  • Object
show all
Defined in:
lib/swarm_memory/optimization/defragmenter.rb

Overview

Defragments memory by finding duplicates, low-quality entries, and archival candidates

This class analyzes memory and suggests optimizations without making changes. Agents must manually review and act on suggestions.

Instance Method Summary collapse

Constructor Details

#initialize(adapter:, embedder: nil) ⇒ Defragmenter

Initialize defragmenter

Parameters:



14
15
16
17
18
# File 'lib/swarm_memory/optimization/defragmenter.rb', line 14

def initialize(adapter:, embedder: nil)
  @adapter = adapter
  @embedder = embedder
  @analyzer = Analyzer.new(adapter: adapter)
end

Instance Method Details

#cleanup_stubs_active(min_age_days: 30, max_hits: 3, dry_run: true) ⇒ String

Clean up old stub files

Parameters:

  • min_age_days (Integer) (defaults to: 30)

    Minimum age for cleanup

  • max_hits (Integer) (defaults to: 3)

    Maximum hits to consider for cleanup

  • dry_run (Boolean) (defaults to: true)

    Preview mode

Returns:

  • (String)

    Result report



408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
# File 'lib/swarm_memory/optimization/defragmenter.rb', line 408

def cleanup_stubs_active(min_age_days: 30, max_hits: 3, dry_run: true)
  stubs = find_stubs_to_cleanup(min_age_days: min_age_days, max_hits: max_hits)

  return "No stubs found for cleanup." if stubs.empty?

  results = []
  freed_bytes = 0

  stubs.each do |stub|
    if dry_run
      results << "Would delete stub: #{stub[:path]} (age: #{stub[:age_days]}d, hits: #{stub[:hits]})"
    else
      freed_bytes += stub[:size]
      @adapter.delete(file_path: stub[:path])
      results << "✓ Deleted stub: #{stub[:path]}"
    end
  end

  format_cleanup_report(results, stubs.size, freed_bytes, dry_run)
end

#compact_active(min_quality_score: 20, min_age_days: 30, max_hits: 0, dry_run: true) ⇒ String

Compact low-value entries (delete permanently)

Parameters:

  • min_quality_score (Integer) (defaults to: 20)

    Minimum quality threshold (0-100)

  • min_age_days (Integer) (defaults to: 30)

    Minimum age

  • max_hits (Integer) (defaults to: 0)

    Maximum hits

  • dry_run (Boolean) (defaults to: true)

    Preview mode

Returns:

  • (String)

    Result report



436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
# File 'lib/swarm_memory/optimization/defragmenter.rb', line 436

def compact_active(min_quality_score: 20, min_age_days: 30, max_hits: 0, dry_run: true)
  entries = @adapter.list
  low_value = []

  entries.each do ||
    entry = @adapter.read_entry(file_path: [:path])

    # Calculate quality from metadata (not content)
    quality = (entry. || {})

    age_days = ((Time.now - entry.updated_at) / 86400).round
    hits = entry.&.dig("hits") || 0

    next if quality >= min_quality_score || age_days < min_age_days || hits > max_hits

    low_value << {
      path: [:path],
      quality: quality,
      age_days: age_days,
      hits: hits,
      size: entry.size,
    }
  end

  return "No low-value entries found for compaction." if low_value.empty?

  results = []
  freed_bytes = 0

  low_value.each do |entry|
    if dry_run
      results << "Would delete: #{entry[:path]} (quality: #{entry[:quality]}, age: #{entry[:age_days]}d, hits: #{entry[:hits]})"
    else
      freed_bytes += entry[:size]
      @adapter.delete(file_path: entry[:path])
      results << "✓ Deleted: #{entry[:path]}"
    end
  end

  format_compact_report(results, low_value.size, freed_bytes, dry_run)
end

#find_archival_candidates(age_days: 90) ⇒ Array<Hash>

Find entries that could be archived (old and unused)

Parameters:

  • age_days (Integer) (defaults to: 90)

    Minimum age in days

Returns:

  • (Array<Hash>)

    Archival candidates



140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
# File 'lib/swarm_memory/optimization/defragmenter.rb', line 140

def find_archival_candidates(age_days: 90)
  entries = @adapter.list
  cutoff_date = Time.now - (age_days * 24 * 60 * 60)

  candidates = entries.select do ||
    [:updated_at] < cutoff_date
  end

  candidates.map do ||
    entry = @adapter.read_entry(file_path: [:path])
     = entry. || {}

    {
      path: [:path],
      title: entry.title,
      age_days: ((Time.now - [:updated_at]) / 86400).round,
      last_verified: ["last_verified"],
      confidence: ["confidence"] || "unknown",
      size: entry.size,
    }
  end.sort_by { |e| -e[:age_days] }
end

#find_archival_candidates_report(age_days: 90) ⇒ String

Generate formatted report for archival candidates

Parameters:

  • age_days (Integer) (defaults to: 90)

    Age threshold

Returns:

  • (String)

    Formatted report



237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
# File 'lib/swarm_memory/optimization/defragmenter.rb', line 237

def find_archival_candidates_report(age_days: 90)
  candidates = find_archival_candidates(age_days: age_days)

  return "No entries older than #{age_days} days found." if candidates.empty?

  report = []
  report << "# Archival Candidates (#{candidates.size} entries older than #{age_days} days)"
  report << ""
  report << "Found #{candidates.size} old entry/entries that could be archived."
  report << ""

  candidates.each do |entry|
    report << "## memory://#{entry[:path]}"
    report << "- Title: #{entry[:title]}"
    report << "- Age: #{entry[:age_days]} days"
    report << "- Last verified: #{entry[:last_verified] || "never"}"
    report << "- Confidence: #{entry[:confidence]}"
    report << "- Size: #{format_bytes(entry[:size])}"
    report << ""
    report << "  **Suggestion:** Review and delete with MemoryDelete if truly obsolete, or use compact action with appropriate thresholds"
    report << ""
  end

  report.join("\n")
end

#find_duplicates(threshold: 0.85) ⇒ Array<Hash>

Find potential duplicate entries

Uses both text similarity (Jaccard) and semantic similarity (embeddings) to find entries that could be merged.

Parameters:

  • threshold (Float) (defaults to: 0.85)

    Similarity threshold (0.0-1.0)

Returns:

  • (Array<Hash>)

    Duplicate pairs with similarity scores



54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
# File 'lib/swarm_memory/optimization/defragmenter.rb', line 54

def find_duplicates(threshold: 0.85)
  entries = @adapter.list
  return [] if entries.size < 2

  duplicates = []
  all_entries = @adapter.all_entries

  # Compare all pairs
  entry_paths = entries.map { |e| e[:path] }
  entry_paths.combination(2).each do |path1, path2|
    entry1 = all_entries[path1]
    entry2 = all_entries[path2]

    # Calculate text similarity (always available)
    text_sim = Search::TextSimilarity.jaccard(entry1.content, entry2.content)

    # Calculate semantic similarity if embeddings available
    semantic_sim = if entry1.embedded? && entry2.embedded?
      Search::TextSimilarity.cosine(entry1.embedding, entry2.embedding)
    end

    # Use highest similarity score
    similarity = [text_sim, semantic_sim].compact.max

    next if similarity < threshold

    duplicates << {
      path1: path1,
      path2: path2,
      similarity: (similarity * 100).round(1),
      text_similarity: (text_sim * 100).round(1),
      semantic_similarity: semantic_sim ? (semantic_sim * 100).round(1) : nil,
      title1: entry1.title,
      title2: entry2.title,
      size1: entry1.size,
      size2: entry2.size,
    }
  end

  duplicates.sort_by { |d| -d[:similarity] }
end

#find_duplicates_report(threshold: 0.85) ⇒ String

Generate formatted report for duplicates

Parameters:

  • threshold (Float) (defaults to: 0.85)

    Similarity threshold

Returns:

  • (String)

    Formatted report



167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
# File 'lib/swarm_memory/optimization/defragmenter.rb', line 167

def find_duplicates_report(threshold: 0.85)
  duplicates = find_duplicates(threshold: threshold)

  return "No duplicate entries found above #{(threshold * 100).round}% similarity." if duplicates.empty?

  report = []
  report << "# Potential Duplicates (#{duplicates.size} pairs)"
  report << ""
  report << "Found #{duplicates.size} pair(s) of similar entries that could potentially be merged."
  report << ""

  duplicates.each_with_index do |dup, index|
    report << "## Pair #{index + 1}: #{dup[:similarity]}% similar"
    report << "- memory://#{dup[:path1]}"
    report << "  Title: \"#{dup[:title1]}\""
    report << "  Size: #{format_bytes(dup[:size1])}"
    report << "- memory://#{dup[:path2]}"
    report << "  Title: \"#{dup[:title2]}\""
    report << "  Size: #{format_bytes(dup[:size2])}"
    report << ""
    report << "  Text similarity: #{dup[:text_similarity]}%"
    report << if dup[:semantic_similarity]
      "  Semantic similarity: #{dup[:semantic_similarity]}%"
    else
      "  Semantic similarity: N/A (no embeddings)"
    end
    report << ""
    report << "  **Suggestion:** Review both entries and consider merging with MemoryEdit"
    report << ""
  end

  report.join("\n")
end

#find_low_quality(confidence_filter: "low") ⇒ Array<Hash>

Find low-quality entries

Parameters:

  • confidence_filter (String) (defaults to: "low")

    Filter level ("low", "medium", "high")

Returns:

  • (Array<Hash>)

    Low-quality entries with issues



100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
# File 'lib/swarm_memory/optimization/defragmenter.rb', line 100

def find_low_quality(confidence_filter: "low")
  entries = @adapter.list
  low_quality = []

  entries.each do ||
    entry = @adapter.read_entry(file_path: [:path])

    # Get metadata from entry (always has string keys from .yml file)
     = entry. || {}

    # Calculate quality score from metadata
    quality = ()

    # Check for issues (all keys are strings)
    issues = []
    issues << "No metadata" if .empty?
    issues << "Confidence: #{["confidence"]}" if should_flag_confidence?(["confidence"], confidence_filter)
    issues << "No type specified" if ["type"].nil?
    issues << "No tags" if (["tags"] || []).empty?
    issues << "No related links" if (["related"] || []).empty?
    issues << "Not embedded" if !entry.embedded? && @embedder

    next if issues.empty?

    low_quality << {
      path: [:path],
      title: entry.title,
      issues: issues,
      confidence: ["confidence"] || "unknown",
      quality_score: quality,
    }
  end

  low_quality.sort_by { |e| e[:quality_score] }
end

#find_low_quality_report(confidence_filter: "low") ⇒ String

Generate formatted report for low-quality entries

Parameters:

  • confidence_filter (String) (defaults to: "low")

    Confidence level filter

Returns:

  • (String)

    Formatted report



205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
# File 'lib/swarm_memory/optimization/defragmenter.rb', line 205

def find_low_quality_report(confidence_filter: "low")
  entries = find_low_quality(confidence_filter: confidence_filter)

  return "No low-quality entries found." if entries.empty?

  report = []
  report << "# Low-Quality Entries (#{entries.size} entries)"
  report << ""
  report << "Found #{entries.size} entry/entries with quality issues."
  report << ""

  entries.each do |entry|
    report << "## memory://#{entry[:path]}"
    report << "- Title: #{entry[:title]}"
    report << "- Quality score: #{entry[:quality_score]}/100"
    report << "- Confidence: #{entry[:confidence]}"
    report << "- Issues:"
    entry[:issues].each do |issue|
      report << "  - #{issue}"
    end
    report << ""
    report << "  **Suggestion:** Add proper frontmatter and metadata with MemoryEdit"
    report << ""
  end

  report.join("\n")
end

Find related entries that should be cross-linked

Finds entry pairs with semantic similarity in the "related" range but NOT duplicates. Uses pure semantic similarity (no keyword boost).

Parameters:

  • min_threshold (Float) (defaults to: 0.60)

    Minimum similarity for relationships (default: 0.60)

  • max_threshold (Float) (defaults to: 0.85)

    Maximum similarity (above = duplicates) (default: 0.85)

Returns:

  • (Array<Hash>)

    Related pairs



271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
# File 'lib/swarm_memory/optimization/defragmenter.rb', line 271

def find_related(min_threshold: 0.60, max_threshold: 0.85)
  entries = @adapter.list
  return [] if entries.size < 2

  related_pairs = []
  all_entries = @adapter.all_entries

  # Compare all pairs
  entry_paths = entries.map { |e| e[:path] }
  entry_paths.combination(2).each do |path1, path2|
    entry1 = all_entries[path1]
    entry2 = all_entries[path2]

    # Skip if no embeddings (need semantic similarity)
    next unless entry1.embedded? && entry2.embedded?

    # Calculate PURE semantic similarity (no keyword boosting for merging)
    semantic_sim = Search::TextSimilarity.cosine(entry1.embedding, entry2.embedding)

    # Must be in the "related" range
    next if semantic_sim < min_threshold
    next if semantic_sim >= max_threshold

    # Check current linking status
    entry1_related = (entry1.["related"] || []).map { |r| r.sub(%r{^memory://}, "") }
    entry2_related = (entry2.["related"] || []).map { |r| r.sub(%r{^memory://}, "") }

    linked_1_to_2 = entry1_related.include?(path2)
    linked_2_to_1 = entry2_related.include?(path1)
    already_linked = linked_1_to_2 && linked_2_to_1

    # Extract metadata
    type1 = entry1.["type"] || "unknown"
    type2 = entry2.["type"] || "unknown"

    related_pairs << {
      path1: path1,
      path2: path2,
      similarity: (semantic_sim * 100).round(1),
      title1: entry1.title,
      title2: entry2.title,
      type1: type1,
      type2: type2,
      already_linked: already_linked,
      linked_1_to_2: linked_1_to_2,
      linked_2_to_1: linked_2_to_1,
    }
  end

  related_pairs.sort_by { |d| -d[:similarity] }
end

Generate formatted report for related entries

Parameters:

  • min_threshold (Float) (defaults to: 0.60)

    Minimum similarity

  • max_threshold (Float) (defaults to: 0.85)

    Maximum similarity

Returns:

  • (String)

    Formatted report



328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
# File 'lib/swarm_memory/optimization/defragmenter.rb', line 328

def find_related_report(min_threshold: 0.60, max_threshold: 0.85)
  pairs = find_related(min_threshold: min_threshold, max_threshold: max_threshold)

  return "No related entry pairs found in #{(min_threshold * 100).round}-#{(max_threshold * 100).round}% similarity range." if pairs.empty?

  report = []
  report << "# Related Entries (#{pairs.size} pairs)"
  report << ""
  report << "Found #{pairs.size} pair(s) of semantically related entries."
  report << "Similarity range: #{(min_threshold * 100).round}-#{(max_threshold * 100).round}% (pure semantic, no keyword boost)"
  report << ""

  pairs.each_with_index do |pair, index|
    report << "## Pair #{index + 1}: #{pair[:similarity]}% similar"
    report << "- memory://#{pair[:path1]} (#{pair[:type1]})"
    report << "  \"#{pair[:title1]}\""
    report << "- memory://#{pair[:path2]} (#{pair[:type2]})"
    report << "  \"#{pair[:title2]}\""
    report << ""

    if pair[:already_linked]
      report << "  ✓ Already linked bidirectionally"
    elsif pair[:linked_1_to_2]
      report << "  → Entry 1 links to Entry 2, but not vice versa"
      report << "  **Suggestion:** Add backward link from Entry 2 to Entry 1"
    elsif pair[:linked_2_to_1]
      report << "  → Entry 2 links to Entry 1, but not vice versa"
      report << "  **Suggestion:** Add backward link from Entry 1 to Entry 2"
    else
      report << "  **Suggestion:** Add bidirectional links to cross-reference these related entries"
    end
    report << ""
  end

  report << "To automatically create missing links, use:"
  report << "  MemoryDefrag(action: \"link_related\", dry_run: true)  # Preview first"
  report << "  MemoryDefrag(action: \"link_related\", dry_run: false) # Execute"
  report << ""

  report.join("\n")
end

#full_analysis(similarity_threshold: 0.85, age_days: 90, confidence_filter: "low") ⇒ String

Run full analysis (all operations)

Parameters:

  • similarity_threshold (Float) (defaults to: 0.85)

    Threshold for duplicate detection

  • age_days (Integer) (defaults to: 90)

    Age threshold for archival

  • confidence_filter (String) (defaults to: "low")

    Confidence level filter

Returns:

  • (String)

    Complete analysis report



33
34
35
36
37
38
39
40
41
42
43
44
45
# File 'lib/swarm_memory/optimization/defragmenter.rb', line 33

def full_analysis(similarity_threshold: 0.85, age_days: 90, confidence_filter: "low")
  report = []
  report << "# Full Memory Defrag Analysis\n"
  report << @analyzer.health_report
  report << "\n---\n"
  report << find_duplicates_report(threshold: similarity_threshold)
  report << "\n---\n"
  report << find_low_quality_report(confidence_filter: confidence_filter)
  report << "\n---\n"
  report << find_archival_candidates_report(age_days: age_days)

  report.join("\n")
end

#full_optimization(dry_run: true) ⇒ String

Full optimization (all operations)

Parameters:

  • dry_run (Boolean) (defaults to: true)

    Preview mode (default: true)

Returns:

  • (String)

    Complete optimization report



546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
# File 'lib/swarm_memory/optimization/defragmenter.rb', line 546

def full_optimization(dry_run: true)
  report = []
  report << "# Full Memory Optimization"
  report << ""
  mode_message = dry_run ? "## DRY RUN MODE - No changes will be made" : "## ACTIVE MODE - Performing optimizations"
  report << mode_message
  report << ""

  # 1. Health baseline
  initial_health = @analyzer.analyze
  report << "Initial health score: #{initial_health[:health_score]}/100"
  report << ""

  # 2. Merge duplicates
  report << "## 1. Merging Duplicates"
  report << merge_duplicates_active(dry_run: dry_run)
  report << ""

  # 3. Cleanup stubs
  report << "## 2. Cleaning Up Stubs"
  report << cleanup_stubs_active(dry_run: dry_run)
  report << ""

  # 4. Compact low-value
  report << "## 3. Compacting Low-Value Entries"
  report << compact_active(dry_run: dry_run)
  report << ""

  # 6. Final health check
  unless dry_run
    final_health = @analyzer.analyze
    report << "## Summary"
    report << "Health score: #{initial_health[:health_score]}#{final_health[:health_score]} (+#{final_health[:health_score] - initial_health[:health_score]})"
  end

  report.join("\n")
end

#health_reportString

Generate health report

Returns:

  • (String)

    Formatted health report



23
24
25
# File 'lib/swarm_memory/optimization/defragmenter.rb', line 23

def health_report
  @analyzer.health_report
end

Create bidirectional links between related entries

Finds related pairs and updates their 'related' metadata to cross-reference each other.

Parameters:

  • min_threshold (Float) (defaults to: 0.60)

    Minimum similarity (default: 0.60)

  • max_threshold (Float) (defaults to: 0.85)

    Maximum similarity (default: 0.85)

  • dry_run (Boolean) (defaults to: true)

    Preview mode (default: true)

Returns:

  • (String)

    Result report



486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
# File 'lib/swarm_memory/optimization/defragmenter.rb', line 486

def link_related_active(min_threshold: 0.60, max_threshold: 0.85, dry_run: true)
  pairs = find_related(min_threshold: min_threshold, max_threshold: max_threshold)

  # Filter to only pairs that need linking
  needs_linking = pairs.reject { |p| p[:already_linked] }

  if needs_linking.empty?
    return "No related entries found that need linking. All similar entries are already cross-referenced."
  end

  report = []
  report << (dry_run ? "# Link Related Entries (DRY RUN)" : "# Link Related Entries")
  report << ""
  report << "Found #{needs_linking.size} pair(s) that should be cross-linked."
  report << ""

  links_created = 0

  needs_linking.each_with_index do |pair, index|
    report << "## Pair #{index + 1}: #{pair[:similarity]}% similar"
    report << "- memory://#{pair[:path1]}"
    report << "- memory://#{pair[:path2]}"
    report << ""

    if dry_run
      # Show what would happen
      if !pair[:linked_1_to_2] && !pair[:linked_2_to_1]
        report << "  Would add bidirectional links:"
        report << "    - Add #{pair[:path2]} to #{pair[:path1]}'s related array"
        report << "    - Add #{pair[:path1]} to #{pair[:path2]}'s related array"
      elsif !pair[:linked_1_to_2]
        report << "  Would add backward link:"
        report << "    - Add #{pair[:path2]} to #{pair[:path1]}'s related array"
      elsif !pair[:linked_2_to_1]
        report << "  Would add backward link:"
        report << "    - Add #{pair[:path1]} to #{pair[:path2]}'s related array"
      end
    else
      # Actually create links
      created = create_bidirectional_links(pair[:path1], pair[:path2], pair[:linked_1_to_2], pair[:linked_2_to_1])
      links_created += created

      report << "  ✓ Created #{created} link(s)"
    end
    report << ""
  end

  report << if dry_run
    "**DRY RUN:** No changes made. Set dry_run=false to execute."
  else
    "**COMPLETED:** Created #{links_created} link(s) across #{needs_linking.size} pairs."
  end

  report.join("\n")
end

#merge_duplicates_active(threshold: 0.85, strategy: :keep_newer, dry_run: true) ⇒ String

Merge duplicate entries

Parameters:

  • threshold (Float) (defaults to: 0.85)

    Similarity threshold

  • strategy (Symbol) (defaults to: :keep_newer)

    Merge strategy (:keep_newer, :keep_larger, :combine)

  • dry_run (Boolean) (defaults to: true)

    If true, show what would be done without doing it

Returns:

  • (String)

    Result report



380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
# File 'lib/swarm_memory/optimization/defragmenter.rb', line 380

def merge_duplicates_active(threshold: 0.85, strategy: :keep_newer, dry_run: true)
  duplicates = find_duplicates(threshold: threshold)

  return "No duplicates found above #{(threshold * 100).round}% similarity." if duplicates.empty?

  results = []
  freed_bytes = 0

  duplicates.each do |pair|
    if dry_run
      results << "Would merge: #{pair[:path2]}#{pair[:path1]} (#{pair[:similarity]}% similar)"
    else
      # Actually merge
      result_info = merge_pair(pair, strategy: strategy)
      freed_bytes += result_info[:freed_bytes]
      results << "✓ Merged: #{result_info[:merged_path]}#{result_info[:kept_path]}"
    end
  end

  format_merge_report(results, duplicates.size, freed_bytes, dry_run)
end