Class: SwarmMemory::Core::Storage

Inherits:
Object
  • Object
show all
Defined in:
lib/swarm_memory/core/storage.rb

Overview

High-level storage orchestration

Coordinates adapter operations, path normalization, embedding generation, and metadata extraction.

Examples:

adapter = Adapters::FilesystemAdapter.new(persist_to: ".swarm/memory.json")
storage = Storage.new(adapter: adapter)
storage.write(file_path: "concepts/ruby", content: "...", title: "Ruby Classes")

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(adapter:, embedder: nil, semantic_weight: nil, keyword_weight: nil) ⇒ Storage

Initialize storage with an adapter

Parameters:

  • adapter (Adapters::Base) —

    Storage adapter

  • embedder (Embeddings::Embedder, nil) (defaults to: nil) —

    Optional embedder for semantic search

  • semantic_weight (Float, nil) (defaults to: nil) —

    Weight for semantic similarity in hybrid search (0.0-1.0)

  • keyword_weight (Float, nil) (defaults to: nil) —

    Weight for keyword matching in hybrid search (0.0-1.0)

Raises:

  • (ArgumentError) 


23
24
25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'lib/swarm_memory/core/storage.rb', line 23

def initialize(adapter:, embedder: nil, semantic_weight: nil, keyword_weight: nil)
  raise ArgumentError, "adapter is required" unless adapter.is_a?(Adapters::Base)

  @adapter = adapter
  @embedder = embedder

  # Create semantic index if embedder is provided
  @semantic_index = if embedder
    index_options = { adapter: adapter, embedder: embedder }
    index_options[:semantic_weight] = semantic_weight if semantic_weight
    index_options[:keyword_weight] = keyword_weight if keyword_weight
    SemanticIndex.new(**index_options)
  end
end

Instance Attribute Details

#adapter ⇒ Object (readonly)

Returns the value of attribute adapter.



15
16
17
 
# File 'lib/swarm_memory/core/storage.rb', line 15

def adapter
  @adapter
end

#semantic_index ⇒ SemanticIndex? (readonly)

Get semantic index for semantic search operations

Returns:

  • (SemanticIndex, nil) —

    Semantic index instance or nil if no embedder



41
42
43
 
# File 'lib/swarm_memory/core/storage.rb', line 41

def semantic_index
  @semantic_index
end

Instance Method Details

#all_entries ⇒ Hash<String, Entry>

Get all entries (for optimization/analysis)

Returns:

  • (Hash<String, Entry>) —

    All entries



246
247
248
 
# File 'lib/swarm_memory/core/storage.rb', line 246

def all_entries
  @adapter.all_entries
end

#clear ⇒ void

This method returns an undefined value.

Clear all entries



225
226
227
 
# File 'lib/swarm_memory/core/storage.rb', line 225

def clear
  @adapter.clear
end

#delete(file_path:) ⇒ void

This method returns an undefined value.

Delete an entry

Parameters:

  • file_path (String) —

    Path to delete



185
186
187
188
 
# File 'lib/swarm_memory/core/storage.rb', line 185

def delete(file_path:)
  normalized_path = PathNormalizer.normalize(file_path)
  @adapter.delete(file_path: normalized_path)
end

#glob(pattern:) ⇒ Array<Hash>

Search by glob pattern

Parameters:

  • pattern (String) —

    Glob pattern

Returns:

  • (Array<Hash>) —

    Matching entries



202
203
204
 
# File 'lib/swarm_memory/core/storage.rb', line 202

def glob(pattern:)
  @adapter.glob(pattern: pattern)
end

#grep(pattern:, case_insensitive: false, output_mode: "files_with_matches", path: nil) ⇒ Array<Hash>

Search by content pattern

Parameters:

  • pattern (String) —

    Regex pattern

  • case_insensitive (Boolean) (defaults to: false) —

    Case-insensitive search

  • output_mode (String) (defaults to: "files_with_matches") —

    Output mode

  • path (String, nil) (defaults to: nil) —

    Optional path prefix filter

Returns:

  • (Array<Hash>) —

    Search results



213
214
215
216
217
218
219
220
 
# File 'lib/swarm_memory/core/storage.rb', line 213

def grep(pattern:, case_insensitive: false, output_mode: "files_with_matches", path: nil)
  @adapter.grep(
    pattern: pattern,
    case_insensitive: case_insensitive,
    output_mode: output_mode,
    path: path,
  )
end

#list(prefix: nil) ⇒ Array<Hash>

List all entries

Parameters:

  • prefix (String, nil) (defaults to: nil) —

    Optional prefix filter

Returns:

  • (Array<Hash>) —

    Entry metadata



194
195
196
 
# File 'lib/swarm_memory/core/storage.rb', line 194

def list(prefix: nil)
  @adapter.list(prefix: prefix)
end

#read(file_path:) ⇒ String

Read content from storage, automatically following stub redirects

Parameters:

  • file_path (String) —

    Path to read from

Returns:

  • (String) —

    Content at the path



107
108
109
110
 
# File 'lib/swarm_memory/core/storage.rb', line 107

def read(file_path:)
  entry = read_entry(file_path: file_path)
  entry.content
end

#read_entry(file_path:, visited: []) ⇒ Entry

Read full entry with metadata, automatically following stub redirects

Stub redirects are created by MemoryDefrag when merging/moving entries. This method transparently follows redirect chains up to 5 levels deep.

Parameters:

  • file_path (String) —

    Path to read from

  • visited (Array<String>) (defaults to: []) —

    Internal: tracks visited paths to detect circular redirects

Returns:

  • (Entry) —

    Full entry object

Raises:

  • (ArgumentError) —

    If path not found, circular redirect detected, or too many redirects



121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
 
# File 'lib/swarm_memory/core/storage.rb', line 121

def read_entry(file_path:, visited: [])
  normalized_path = PathNormalizer.normalize(file_path)

  # Detect circular redirects immediately
  if visited.include?(normalized_path)
    cycle = visited + [normalized_path]
    raise ArgumentError,
      "Circular redirect detected in memory storage: #{cycle.join(" → ")}\n\n" \
        "This indicates corrupted stub files. Please run MemoryDefrag to repair:\n  " \
        "MemoryDefrag(action: \"analyze\")"
  end

  # Check depth limit (prevent infinite chains)
  if visited.size >= 5
    chain = visited + [normalized_path]
    raise ArgumentError,
      "Memory redirect chain too deep (>5 redirects): #{chain.join(" → ")}\n\n" \
        "This indicates fragmented memory storage. Please run maintenance:\n  " \
        "MemoryDefrag(action: \"full\", dry_run: true)  # Preview first\n  " \
        "MemoryDefrag(action: \"full\", dry_run: false) # Execute"
  end

  # Read entry from adapter
  begin
    entry = @adapter.read_entry(file_path: normalized_path)
  rescue ArgumentError
    # If this is a redirect target that doesn't exist, provide helpful error
    if visited.empty?
      # Not a redirect, just re-raise original error
      raise
    else
      original_path = visited.first
      raise ArgumentError,
        "memory://#{original_path} was redirected to memory://#{normalized_path}, but the target was not found.\n\n" \
          "The original entry may have been merged or moved incorrectly. " \
          "Run MemoryDefrag to identify and fix broken redirects:\n  " \
          "MemoryDefrag(action: \"analyze\")"
    end
  end

  # Check if this is a stub redirect
  if entry.metadata && entry.metadata["stub"] == true
    redirect_target = entry.metadata["redirect_to"]

    # Validate redirect target exists
    if redirect_target.nil? || redirect_target.strip.empty?
      raise ArgumentError,
        "memory://#{normalized_path} is a stub with invalid redirect metadata.\n\n" \
          "This should never happen (stubs are created by MemoryDefrag). " \
          "The stub file may be corrupted. Please report this as a bug."
    end

    # Follow redirect recursively, tracking visited paths
    return read_entry(file_path: redirect_target, visited: visited + [normalized_path])
  end

  # Not a stub, return the entry
  entry
end

#size ⇒ Integer

Get number of entries

Returns:

  • (Integer) —

    Entry count



239
240
241
 
# File 'lib/swarm_memory/core/storage.rb', line 239

def size
  @adapter.size
end

#total_size ⇒ Integer

Get total storage size

Returns:

  • (Integer) —

    Size in bytes



232
233
234
 
# File 'lib/swarm_memory/core/storage.rb', line 232

def total_size
  @adapter.total_size
end

#write(file_path:, content:, title:, metadata: nil, generate_embedding: nil) ⇒ Entry

Write content to storage

Parameters:

  • file_path (String) —

    Path to store content (with .md extension)

  • content (String) —

    Content to store (pure markdown)

  • title (String) —

    Brief title

  • metadata (Hash, nil) (defaults to: nil) —

    Optional metadata

  • generate_embedding (Boolean) (defaults to: nil) —

    Whether to generate embedding (default: true if embedder present)

Returns:

  • (Entry) —

    The created entry



51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
 
# File 'lib/swarm_memory/core/storage.rb', line 51

def write(file_path:, content:, title:, metadata: nil, generate_embedding: nil)
  # Normalize path
  normalized_path = PathNormalizer.normalize(file_path)

  # Generate embedding if requested and embedder available
  embedding = nil
  should_embed = generate_embedding.nil? ? !@embedder.nil? : generate_embedding

  if should_embed && @embedder
    begin
      # Build searchable text for better semantic matching
      # Uses title + tags + first paragraph instead of full content
      searchable_text = build_searchable_text(content, title, metadata)

      # ALWAYS emit to LogStream (create if needed for debugging)
      # This ensures we can see what's being embedded
      begin
        if defined?(SwarmSDK::LogStream)
          SwarmSDK::LogStream.emit(
            type: "memory_embedding_generated",
            file_path: normalized_path,
            title: title,
            searchable_text_length: searchable_text.length,
            searchable_text_preview: searchable_text.slice(0, 300),
            full_searchable_text: searchable_text,
            metadata_tags: metadata&.dig("tags"),
            metadata_domain: metadata&.dig("domain"),
          )
        end
      rescue StandardError => e
        # Don't fail if logging fails
        warn("Failed to log embedding: #{e.message}")
      end

      embedding = @embedder.embed(searchable_text)
    rescue StandardError => e
      # Don't fail write if embedding generation fails
      warn("Warning: Failed to generate embedding for #{normalized_path}: #{e.message}")
      embedding = nil
    end
  end

  # Write to adapter (metadata passed from tool parameters)
  @adapter.write(
    file_path: normalized_path,
    content: content,
    title: title,
    embedding: embedding,
    metadata: metadata,
  )
end