Class: SpaCyModel
- Inherits:
-
VectorModel
- Object
- VectorModel
- SpaCyModel
- Defined in:
- lib/rbbt/vector/model/spaCy.rb
Instance Attribute Summary collapse
-
#config ⇒ Object
Returns the value of attribute config.
Attributes inherited from VectorModel
#balance, #bar, #directory, #eval_model, #extract_features, #factor_levels, #features, #init_model, #labels, #model, #model_options, #model_path, #names, #post_process, #train_model
Class Method Summary collapse
Instance Method Summary collapse
-
#initialize(dir, config, categories = %w(positive negative), lang = 'en_core_web_md') ⇒ SpaCyModel
constructor
A new instance of SpaCyModel.
Methods inherited from VectorModel
R_eval, R_run, R_train, #__load_method, #add, #add_list, #balance_labels, #clear, #cross_validation, #eval, #eval_list, f1_metrics, #init, #run, #save_models, #train
Constructor Details
#initialize(dir, config, categories = %w(positive negative), lang = 'en_core_web_md') ⇒ SpaCyModel
Returns a new instance of SpaCyModel.
13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 |
# File 'lib/rbbt/vector/model/spaCy.rb', line 13 def initialize(dir, config, categories = %w(positive negative), lang = 'en_core_web_md') @config = case when Path === config config.read when Misc.is_filename?(config) Open.read(config) when (Misc.is_filename?(config, false) && Rbbt.share.spaCy.cpu[config].exists?) Rbbt.share.spaCy.cpu[config].read when (Misc.is_filename?(config, false) && Rbbt.share.spaCy[config].exists?) Rbbt.share.spaCy[config].read else config end Log.debug "SpaCy model loading config from: #{@config}" @lang = lang super(dir) @train_model = Proc.new do |features, labels| texts = features docs = [] unique_labels = labels.uniq tmpconfig = File.join(@model_path, 'config') tmptrain = File.join(@model_path, 'train.spacy') SpaCy.config(@config, tmpconfig) = (features.length, "Training documents into spacy format") SpaCyModel.spacy do nlp = SpaCy.nlp(lang) docs = [] RbbtPython.iterate nlp.pipe(texts.zip(labels), as_tuples: true), :bar => do |doc,label| unique_labels.each do |other_label| next if other_label == label doc.cats[other_label] = false end doc.cats[label] = true docs << doc end doc_bin = spacy.tokens.DocBin.new(docs: docs) doc_bin.to_disk(tmptrain) end gpu = Rbbt::Config.get('gpu_id', :spacy, :spacy_train, :default => 0) CMD.cmd_log(:spacy, "train #{tmpconfig} --output #{@model_path} --paths.train #{tmptrain} --paths.dev #{tmptrain}", "--gpu-id" => gpu) end @eval_model = Proc.new do |features,list| texts = features texts = [texts] unless list model_path = @model_path docs = [] = (features.length, "Evaluating model") SpaCyModel.spacy do gpu = Rbbt::Config.get('gpu_id', :spacy, :spacy_train, :default => 0) gpu = gpu.to_i if gpu && gpu != "" spacy.require_gpu(gpu) if gpu nlp = spacy.load("#{model_path}/model-best") docs = nlp.pipe(texts) RbbtPython.collect docs, :bar => do |d| Misc.timeout_insist(20) do d.cats.sort_by{|l,v| v.to_f || 0 }.last.first end end end end end |
Instance Attribute Details
#config ⇒ Object
Returns the value of attribute config.
5 6 7 |
# File 'lib/rbbt/vector/model/spaCy.rb', line 5 def config @config end |
Class Method Details
.spacy(&block) ⇒ Object
7 8 9 10 11 |
# File 'lib/rbbt/vector/model/spaCy.rb', line 7 def self.spacy(&block) RbbtPython.run "spacy" do RbbtPython.module_eval(&block) end end |