Class: SwarmMemory::Adapters::FilesystemAdapter

Inherits:
Base
  • Object
show all
Defined in:
lib/swarm_memory/adapters/filesystem_adapter.rb

Overview

Real filesystem adapter using .md/.yml file pairs

Architecture:

  • Content stored in .md files (markdown)
  • Metadata stored in .yml files (tags, confidence, hits)
  • Embeddings stored in .emb files (binary, optional)
  • Paths flattened with -- separator for Git-friendly structure
  • Stubs for merged/moved entries with auto-redirect
  • Hit tracking for access patterns

Example on disk:

.swarm/memory/
├── concepts--ruby--classes.md       (content)
├── concepts--ruby--classes.yml      (metadata)
├── concepts--ruby--classes.emb      (embedding, optional)
└── _stubs/
  ├── old-ruby-intro.md            (stub: "# merged → concepts--ruby--classes")
  └── old-ruby-intro.yml           (metadata with stub: true)

Constant Summary collapse

STUB_MARKERS =

Stub markers

["# merged →", "# moved →"].freeze
VIRTUAL_ENTRIES =

Virtual built-in entries that always exist without taking storage space These are meta-skills and resources available to all agents Mapped as: memory_path => gem_file_basename

{
  "skill/meta/deep-learning.md" => "meta/deep-learning",
}.freeze

Constants inherited from Base

Base::MAX_ENTRY_SIZE, Base::MAX_TOTAL_SIZE

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(directory:) ⇒ FilesystemAdapter

Initialize filesystem adapter with directory

Parameters:

  • directory (String) —

    Directory path for storage (REQUIRED)

Raises:

  • (ArgumentError) —

    If directory is not provided



38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
 
# File 'lib/swarm_memory/adapters/filesystem_adapter.rb', line 38

def initialize(directory:)
  super()
  raise ArgumentError, "directory is required for FilesystemAdapter" if directory.nil? || directory.to_s.strip.empty?

  @directory = File.expand_path(directory)
  @semaphore = Async::Semaphore.new(1) # Fiber-aware concurrency control
  @total_size = 0

  # Create directory if it doesn't exist
  FileUtils.mkdir_p(@directory)

  # Lock file for cross-process synchronization
  @lock_file_path = File.join(@directory, ".lock")

  # Build in-memory index on boot (for fast lookups)
  @index = build_index
end

Instance Attribute Details

#total_size ⇒ Integer (readonly)

Get current total size

Returns:

  • (Integer) —

    Total size in bytes



380
381
382
 
# File 'lib/swarm_memory/adapters/filesystem_adapter.rb', line 380

def total_size
  @total_size
end

Instance Method Details

#all_entries ⇒ Hash<String, Core::Entry>

Get all entries (for optimization/analysis)

Returns:



392
393
394
395
396
397
398
399
400
401
402
403
 
# File 'lib/swarm_memory/adapters/filesystem_adapter.rb', line 392

def all_entries
  entries = {}

  @index.each do |logical_path, _index_data|
    entries[logical_path] = read_entry(file_path: logical_path)
  rescue ArgumentError
    # Skip entries that can't be read
    next
  end

  entries
end

#clear ⇒ void

This method returns an undefined value.

Clear all entries



363
364
365
366
367
368
369
370
371
372
373
374
375
 
# File 'lib/swarm_memory/adapters/filesystem_adapter.rb', line 363

def clear
  with_write_lock do
    @semaphore.acquire do
      # Delete all .md, .yml, .emb files
      Dir.glob(File.join(@directory, "**/*.{md,yml,emb}")).each do |file|
        File.delete(file)
      end

      @total_size = 0
      @index = {}
    end
  end
end

#delete(file_path:) ⇒ void

This method returns an undefined value.

Delete entry from filesystem

Parameters:

  • file_path (String) —

    Logical path with .md extension



228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
 
# File 'lib/swarm_memory/adapters/filesystem_adapter.rb', line 228

def delete(file_path:)
  with_write_lock do
    @semaphore.acquire do
      raise ArgumentError, "file_path is required" if file_path.nil? || file_path.to_s.strip.empty?

      # Strip .md extension
      base_path = file_path.sub(/\.md\z/, "")
      disk_path = base_path
      md_file = File.join(@directory, "#{disk_path}.md")

      raise ArgumentError, "memory://#{file_path} not found" unless File.exist?(md_file)

      # Get size before deletion
      entry_size = get_entry_size(file_path)

      # Delete all related files
      File.delete(md_file) if File.exist?(md_file)
      File.delete(File.join(@directory, "#{disk_path}.yaml")) if File.exist?(File.join(@directory, "#{disk_path}.yaml"))
      File.delete(File.join(@directory, "#{disk_path}.emb")) if File.exist?(File.join(@directory, "#{disk_path}.emb"))

      # Update total size
      @total_size -= entry_size

      # Update index
      @index.delete(file_path)
    end
  end
end

#glob(pattern:) ⇒ Array<Hash>

Search by glob pattern

Parameters:

  • pattern (String) —

    Glob pattern (e.g., "concepts/**/*.md")

Returns:

  • (Array<Hash>) —

    Matching entries

Raises:

  • (ArgumentError) 


292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
 
# File 'lib/swarm_memory/adapters/filesystem_adapter.rb', line 292

def glob(pattern:)
  raise ArgumentError, "pattern is required" if pattern.nil? || pattern.to_s.strip.empty?

  # Normalize pattern to ensure we only match .md files
  # Standard glob behavior - just add .md extension intelligently
  normalized_pattern = if pattern.end_with?("**")
    # fact/** → fact/**/*.md (recursive match of all .md files)
    "#{pattern}/*.md"
  elsif pattern.end_with?("*")
    # fact/* → fact/*.md (direct children .md files only)
    "#{pattern}.md"
  elsif pattern.end_with?(".md")
    # Already has .md, use as-is
    pattern
  else
    # No wildcard or extension, add .md
    "#{pattern}.md"
  end

  # Use native Dir.glob with hierarchical paths - efficient!
  glob_pattern = File.join(@directory, normalized_pattern)
  md_files = Dir.glob(glob_pattern).reject { |f| stub_file?(f) }

  results = md_files.map do |md_file|
    # Calculate logical path relative to @directory
    relative_path = md_file.sub("#{@directory}/", "")

    yaml_file = md_file.sub(".md", ".yml")
    yaml_data = File.exist?(yaml_file) ? YAML.load_file(yaml_file, permitted_classes: [Time, Date, Symbol]) : {}

    {
      path: relative_path,
      title: yaml_data["title"] || "Untitled",
      size: File.size(md_file),
      updated_at: parse_time(yaml_data["updated_at"]) || File.mtime(md_file),
    }
  end

  results.sort_by { |e| -e[:updated_at].to_f }
end

#grep(pattern:, case_insensitive: false, output_mode: "files_with_matches", path: nil) ⇒ Array<Hash>

Search by content pattern

Fast path: grep .yml files first (metadata) Fallback: grep .md files (content)

Parameters:

  • pattern (String) —

    Regex pattern

  • case_insensitive (Boolean) (defaults to: false) —

    Case-insensitive search

  • output_mode (String) (defaults to: "files_with_matches") —

    Output mode

Returns:

  • (Array<Hash>) —

    Results

Raises:

  • (ArgumentError) 


342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
 
# File 'lib/swarm_memory/adapters/filesystem_adapter.rb', line 342

def grep(pattern:, case_insensitive: false, output_mode: "files_with_matches", path: nil)
  raise ArgumentError, "pattern is required" if pattern.nil? || pattern.to_s.strip.empty?

  flags = case_insensitive ? Regexp::IGNORECASE : 0
  regex = Regexp.new(pattern, flags)

  case output_mode
  when "files_with_matches"
    grep_files_with_matches(regex, path)
  when "content"
    grep_with_content(regex, path)
  when "count"
    grep_with_count(regex, path)
  else
    raise ArgumentError, "Invalid output_mode: #{output_mode}"
  end
end

#list(prefix: nil) ⇒ Array<Hash>

List all entries

Parameters:

  • prefix (String, nil) (defaults to: nil) —

    Filter by prefix

Returns:

  • (Array<Hash>) —

    Entry metadata



261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
 
# File 'lib/swarm_memory/adapters/filesystem_adapter.rb', line 261

def list(prefix: nil)
  # Find all .md files (excluding stubs)
  md_files = Dir.glob(File.join(@directory, "**/*.md"))
    .reject { |f| stub_file?(f) }

  entries = md_files.map do |md_file|
    # Calculate logical path relative to @directory
    logical_path = md_file.sub("#{@directory}/", "")
    base_logical_path = logical_path.sub(/\.md\z/, "")

    # Filter by prefix if provided (strip .md for comparison)
    next if prefix && !base_logical_path.start_with?(prefix.sub(/\.md\z/, ""))

    yaml_file = md_file.sub(".md", ".yml")
    yaml_data = File.exist?(yaml_file) ? YAML.load_file(yaml_file, permitted_classes: [Time, Date, Symbol]) : {}

    {
      path: logical_path,
      title: yaml_data["title"] || "Untitled",
      size: yaml_data["size"] || File.size(md_file),
      updated_at: parse_time(yaml_data["updated_at"]) || File.mtime(md_file),
    }
  end.compact

  entries.sort_by { |e| e[:path] }
end

#read(file_path:) ⇒ String

Read content from filesystem

Parameters:

  • file_path (String) —

    Logical path with .md extension

Returns:

  • (String) —

    Content

Raises:

  • (ArgumentError) 


156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
 
# File 'lib/swarm_memory/adapters/filesystem_adapter.rb', line 156

def read(file_path:)
  raise ArgumentError, "file_path is required" if file_path.nil? || file_path.to_s.strip.empty?

  # Check for virtual built-in entries first
  if VIRTUAL_ENTRIES.key?(file_path)
    entry = load_virtual_entry(file_path)
    return entry.content
  end

  # Strip .md extension
  base_path = file_path.sub(/\.md\z/, "")
  disk_path = base_path
  md_file = File.join(@directory, "#{disk_path}.md")

  raise ArgumentError, "memory://#{file_path} not found" unless File.exist?(md_file)

  content = File.read(md_file)

  # Increment hit counter
  increment_hits(file_path)

  content
end

#read_entry(file_path:) ⇒ Core::Entry

Read full entry with all metadata

Parameters:

  • file_path (String) —

    Logical path with .md extension

Returns:

Raises:

  • (ArgumentError) 


184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
 
# File 'lib/swarm_memory/adapters/filesystem_adapter.rb', line 184

def read_entry(file_path:)
  raise ArgumentError, "file_path is required" if file_path.nil? || file_path.to_s.strip.empty?

  # Check for virtual built-in entries first
  if VIRTUAL_ENTRIES.key?(file_path)
    return load_virtual_entry(file_path)
  end

  # Strip .md extension
  base_path = file_path.sub(/\.md\z/, "")
  disk_path = base_path
  md_file = File.join(@directory, "#{disk_path}.md")
  yaml_file = File.join(@directory, "#{disk_path}.yml")

  raise ArgumentError, "memory://#{file_path} not found" unless File.exist?(md_file)

  content = File.read(md_file)

  # Read metadata
  yaml_data = File.exist?(yaml_file) ? YAML.load_file(yaml_file, permitted_classes: [Time, Date, Symbol]) : {}

  # Read embedding if exists
  emb_file = File.join(@directory, "#{disk_path}.emb")
  embedding = if File.exist?(emb_file)
    File.read(emb_file).unpack("f*")
  end

  # Increment hit counter
  increment_hits(file_path)

  Core::Entry.new(
    content: content,
    title: yaml_data["title"] || "Untitled",
    updated_at: parse_time(yaml_data["updated_at"]) || Time.now,
    size: yaml_data["size"] || content.bytesize,
    embedding: embedding,
    metadata: yaml_data["metadata"],
  )
end

#semantic_search(embedding:, top_k: 10, threshold: 0.0) ⇒ Array<Hash>

Semantic search by embedding vector

Searches all entries with embeddings and returns those similar to the query. Results are sorted by cosine similarity in descending order.

Examples:

results = adapter.semantic_search(
  embedding: query_embedding,
  top_k: 5,
  threshold: 0.65
)

Parameters:

  • embedding (Array<Float>) —

    Query embedding vector

  • top_k (Integer) (defaults to: 10) —

    Number of results to return

  • threshold (Float) (defaults to: 0.0) —

    Minimum similarity score (0.0-1.0)

Returns:

  • (Array<Hash>) —

    Results with similarity scores



421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
 
# File 'lib/swarm_memory/adapters/filesystem_adapter.rb', line 421

def semantic_search(embedding:, top_k: 10, threshold: 0.0)
  results = []

  # Iterate all entries in the index
  @index.each do |logical_path, index_data|
    # Load embedding file
    emb_file = File.join(@directory, "#{index_data[:disk_path]}.emb")
    next unless File.exist?(emb_file)

    # Read and unpack embedding
    entry_embedding = File.read(emb_file).unpack("f*")

    # Compute cosine similarity
    similarity = cosine_similarity(embedding, entry_embedding)
    next if similarity < threshold

    # Load metadata from YAML
    yaml_file = File.join(@directory, "#{index_data[:disk_path]}.yml")
    yaml_data = if File.exist?(yaml_file)
      YAML.load_file(yaml_file, permitted_classes: [Time, Date, Symbol])
    else
      {}
    end

    # Build result
    results << {
      path: logical_path,
      similarity: similarity,
      title: index_data[:title],
      size: index_data[:size],
      updated_at: index_data[:updated_at],
      metadata: yaml_data["metadata"],
    }
  end

  # Sort by similarity descending, return top K
  results.sort_by { |r| -r[:similarity] }.take(top_k)
end

#size ⇒ Integer

Get number of entries

Returns:

  • (Integer) —

    Number of entries



385
386
387
 
# File 'lib/swarm_memory/adapters/filesystem_adapter.rb', line 385

def size
  @index.size
end

#write(file_path:, content:, title:, embedding: nil, metadata: nil) ⇒ Core::Entry

Write content to filesystem

Parameters:

  • file_path (String) —

    Logical path (e.g., "concepts/ruby/classes")

  • content (String) —

    Content to store

  • title (String) —

    Brief title

  • embedding (Array<Float>, nil) (defaults to: nil) —

    Optional embedding vector

  • metadata (Hash, nil) (defaults to: nil) —

    Optional metadata

Returns:



64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
 
# File 'lib/swarm_memory/adapters/filesystem_adapter.rb', line 64

def write(file_path:, content:, title:, embedding: nil, metadata: nil)
  with_write_lock do
    @semaphore.acquire do
      raise ArgumentError, "file_path is required" if file_path.nil? || file_path.to_s.strip.empty?
      raise ArgumentError, "content is required" if content.nil?
      raise ArgumentError, "title is required" if title.nil? || title.to_s.strip.empty?

      # Content is stored as-is (no frontmatter extraction)
      # Metadata comes from tool parameters, not from content
      content_size = content.bytesize

      # Ensure all metadata keys are strings
      stringified_metadata = metadata ? Utils.stringify_keys(metadata) : {}

      # Check entry size limit
      if content_size > MAX_ENTRY_SIZE
        raise ArgumentError, "Content exceeds maximum size (#{format_bytes(MAX_ENTRY_SIZE)}). " \
          "Current: #{format_bytes(content_size)}"
      end

      # Calculate new total size
      existing_size = get_entry_size(file_path)
      new_total_size = @total_size - existing_size + content_size

      # Check total size limit
      if new_total_size > MAX_TOTAL_SIZE
        raise ArgumentError, "Memory storage full (#{format_bytes(MAX_TOTAL_SIZE)} limit). " \
          "Current: #{format_bytes(@total_size)}, " \
          "Would be: #{format_bytes(new_total_size)}. " \
          "Clear old entries or use smaller content."
      end

      # Strip .md extension for disk storage
      # "concepts/ruby/classes.md" → "concepts/ruby/classes"
      base_path = file_path.sub(/\.md\z/, "")
      disk_path = base_path

      # 1. Write content to .md file (stored exactly as provided)
      md_file = File.join(@directory, "#{disk_path}.md")
      FileUtils.mkdir_p(File.dirname(md_file))
      File.write(md_file, content)

      # 2. Write metadata to .yml file
      yaml_file = File.join(@directory, "#{disk_path}.yml")
      existing_hits = read_yaml_field(yaml_file, :hits) || 0

      yaml_data = {
        title: title,
        file_path: file_path, # Logical path with .md extension
        updated_at: Time.now,
        size: content_size,
        hits: existing_hits, # Preserve hit count
        metadata: stringified_metadata, # Metadata from tool parameters
        embedding_checksum: embedding ? checksum(embedding) : nil,
      }
      # Convert symbol keys to strings for clean YAML output
      File.write(yaml_file, YAML.dump(Utils.stringify_keys(yaml_data)))

      # 3. Write embedding to .emb file (binary, optional)
      if embedding
        emb_file = File.join(@directory, "#{disk_path}.emb")
        File.write(emb_file, embedding.pack("f*"))
      end

      # Update total size
      @total_size = new_total_size

      # Update index
      @index[file_path] = {
        disk_path: disk_path,
        title: title,
        size: content_size,
        updated_at: Time.now,
      }

      # Return entry object
      Core::Entry.new(
        content: content,
        title: title,
        updated_at: Time.now,
        size: content_size,
        embedding: embedding,
        metadata: stringified_metadata,
      )
    end
  end
end