Class: Clip::Model

Inherits:
Object
  • Object
show all
Defined in:
lib/clip/model.rb

Instance Method Summary collapse

Constructor Details

#initialize(textual_model_path: ".clip_models/textual.onnx", visual_model_path: ".clip_models/visual.onnx", tokenizer: Clip::Tokenizer.new, image_preprocessor: Clip::ImagePreprocessor.new, download_models: true, download_dir: ".clip_models") ⇒ Model

Returns a new instance of Model.



5
6
7
8
9
10
11
12
13
14
15
16
17
18
 
# File 'lib/clip/model.rb', line 5

def initialize(
  textual_model_path: ".clip_models/textual.onnx",
  visual_model_path: ".clip_models/visual.onnx",
  tokenizer: Clip::Tokenizer.new,
  image_preprocessor: Clip::ImagePreprocessor.new,
  download_models: true,
  download_dir: ".clip_models"
)
  @textual_model_path = textual_model_path
  @visual_model_path = visual_model_path
  Clip.download_models(download_dir) if download_models && !Clip.models_exist?(textual_model_path: textual_model_path, visual_model_path: visual_model_path)
  @tokenizer = tokenizer
  @image_preprocessor = image_preprocessor
end

Instance Method Details

#encode_image(image) ⇒ Object 



25
26
27
28
 
# File 'lib/clip/model.rb', line 25

def encode_image(image)
  image = image_preprocessor.preprocess(image).to_a
  image_model.predict({ input: [ image ] })["output"].first
end

#encode_text(text) ⇒ Object 



20
21
22
23
 
# File 'lib/clip/model.rb', line 20

def encode_text(text)
  tokens = tokenizer.encode(text)
  text_model.predict({ input: [ tokens ] })["output"].first
end

#image_model ⇒ Object 



34
35
36
 
# File 'lib/clip/model.rb', line 34

def image_model
  @image_model ||= OnnxRuntime::Model.new(visual_model_path)
end

#text_model ⇒ Object 



30
31
32
 
# File 'lib/clip/model.rb', line 30

def text_model
  @text_model ||= OnnxRuntime::Model.new(textual_model_path)
end