Module: CrawlabRubySdk

Defined in:
lib/crawlab_ruby_sdk.rb,
lib/crawlab_ruby_sdk/version.rb

Defined Under Namespace

Classes: Error

Constant Summary collapse

VERSION = 
"0.2.3"

Class Method Summary collapse

Class Method Details

._save(sub_client, items = []) ⇒ Object 



94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
 
# File 'lib/crawlab_ruby_sdk.rb', line 94

def self._save(sub_client,items=[])
  records = []
  task_id = get_task_id
  if task_id == nil || task_id == ""
    return
  end

  items.each do |item|
    item["_tid"] = task_id
    records << item
  end

  data = {task_id: task_id,data:records}.to_json.b

  # puts "=====#{{task_id: task_id,data:records}.to_json}==="

  msg = Grpc::StreamMessage.new(code:3,data:data)

  sub_client.Send([msg]) 
end

.get_table_name(table_name = "") ⇒ Object 



131
132
133
134
135
136
137
138
139
 
# File 'lib/crawlab_ruby_sdk.rb', line 131

def self.get_table_name(table_name="")
  table_names = Verify.table_names
  if ENV["TABLE_NAME"] != nil && ENV["TABLE_NAME"] != "" && table_names.include?(ENV["TABLE_NAME"])
    table_name = ENV["TABLE_NAME"]
    return table_name
  end 

  return table_name
end

.get_task_id ⇒ Object 



115
116
117
118
119
120
121
 
# File 'lib/crawlab_ruby_sdk.rb', line 115

def self.get_task_id
  task_id = ENV["CRAWLAB_TASK_ID"]
  if task_id == nil || task_id == ""
    task_id = "6486e256fa1cb07a47c09adf"
  end
  return task_id
end

.save(sub_client, items = []) ⇒ Object 



77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
 
# File 'lib/crawlab_ruby_sdk.rb', line 77

def self.save(sub_client,items = [])
  results = []

  items.each do |item|
    results << item

    if results.size >= 50
      _save(sub_client,results)
    end
    
  end

  if results.size > 0
    _save(sub_client,results)
  end
end

.save_file_stream_to_oss(oss_path, stream) ⇒ Object 



127
128
129
 
# File 'lib/crawlab_ruby_sdk.rb', line 127

def self.save_file_stream_to_oss(oss_path,stream)
  OssServerClient.new.send_stream(oss_path,stream)
end

.save_file_to_oss(oss_path, file_path) ⇒ Object 



123
124
125
 
# File 'lib/crawlab_ruby_sdk.rb', line 123

def self.save_file_to_oss(oss_path,file_path)
  OssServerClient.new.send(oss_path,file_path)
end

.save_item(item = {}, table_name = "") ⇒ Object 



31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/crawlab_ruby_sdk.rb', line 31

def self.save_item(item={},table_name="")
  address = ENV["CRAWLAB_GRPC_ADDRESS"]
  if address==nil || address == ""
    address = "localhost:9666"
  end  

  auth = ENV["CRAWLAB_GRPC_AUTH_KEY"]
  if auth==nil || auth == ""
    auth = "Crawlab2021!"
  end  

  table_name = get_table_name(table_name)
  if !Verify.IsVerified?([item],table_name)
    return
  end
  
  client = TaskServiceClient.new(address,auth)

  sub_client = client.subscribe

  save(sub_client,[item])
end

.save_items(items = [], table_name = "") ⇒ Object 



54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/crawlab_ruby_sdk.rb', line 54

def self.save_items(items=[],table_name="")
  address = ENV["CRAWLAB_GRPC_ADDRESS"]
  if address==nil || address == ""
    address = "localhost:9666"
  end  

  auth = ENV["CRAWLAB_GRPC_AUTH_KEY"]
  if auth==nil || auth == ""
    auth = "Crawlab2021!"
  end  

  table_name = get_table_name(table_name)
  if !Verify.IsVerified?(items,table_name)
    return
  end    
  
  client = TaskServiceClient.new(address,auth)

  sub_client = client.subscribe

  save(sub_client,items)    
end