Module: ScoutPython

Extended by:
PyCall::Import
Defined in:
lib/scout/python.rb,
lib/scout/python/run.rb,
lib/scout/python/util.rb,
lib/scout/python/paths.rb,
lib/scout/python/script.rb

Defined Under Namespace

Classes: Binding, ScoutPythonException

Constant Summary collapse

MUTEX = 
Mutex.new

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.paths ⇒ Object

Returns the value of attribute paths.



3
4
5
 
# File 'lib/scout/python/paths.rb', line 3

def paths
  @paths
end

.thread ⇒ Object

Returns the value of attribute thread.



3
4
5
 
# File 'lib/scout/python/run.rb', line 3

def thread
  @thread
end

Class Method Details

.add_path(path) ⇒ Object 



9
10
11
 
# File 'lib/scout/python/paths.rb', line 9

def self.add_path(path)
  self.paths << path
end

.add_paths(paths) ⇒ Object 



13
14
15
 
# File 'lib/scout/python/paths.rb', line 13

def self.add_paths(paths)
  self.paths.concat paths
end

.binding_run(binding = nil, *args, &block) ⇒ Object 



153
154
155
156
 
# File 'lib/scout/python.rb', line 153

def self.binding_run(binding = nil, *args, &block)
  binding = new_binding
  binding.instance_exec *args, &block
end

.call_method(module_name, method_name, *args) ⇒ Object 



46
47
48
 
# File 'lib/scout/python.rb', line 46

def self.call_method(module_name, method_name, *args)
  ScoutPython.import_method(module_name, method_name).call(*args)
end

.class_new_obj(module_name, class_name, args = {}) ⇒ Object 



62
63
64
 
# File 'lib/scout/python.rb', line 62

def self.class_new_obj(module_name, class_name, args={})
  ScoutPython.get_class(module_name, class_name).new(**args)
end

.collect(iterator, options = {}, &block) ⇒ Object 



140
141
142
143
144
145
146
147
 
# File 'lib/scout/python.rb', line 140

def self.collect(iterator, options = {}, &block)
  acc = []
  self.iterate(iterator, options) do |elem|
    res = block.call elem
    acc << res
  end
  acc
end

.df2tsv(tuple, options = {}) ⇒ Object 



19
20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/scout/python/util.rb', line 19

def self.df2tsv(tuple, options = {})
  options = IndiferentHash.add_defaults options, :type => :list
  IndiferentHash.setup options
  tsv = TSV.setup({}, options)
  tsv.key_field = options[:key_field] || tuple.columns.name.to_s
  tsv.fields = py2ruby_a(tuple.columns.values)
  keys = py2ruby_a(tuple.index.values)
  PyCall.len(tuple.index).times do |i|
    k = keys[i]
    tsv[k] = py2ruby_a(tuple.values[i])
  end
  tsv
end

.exec(script) ⇒ Object 



66
67
68
 
# File 'lib/scout/python.rb', line 66

def self.exec(script)
  PyCall.exec(script)
end

.get_class(module_name, class_name) ⇒ Object 



57
58
59
60
 
# File 'lib/scout/python.rb', line 57

def self.get_class(module_name, class_name)
  mod = get_module(module_name)
  mod.send(class_name)
end

.get_module(module_name) ⇒ Object 



50
51
52
53
54
55
 
# File 'lib/scout/python.rb', line 50

def self.get_module(module_name)
  init_scout
  save_module_name = module_name.to_s.gsub(".", "_")
  ScoutPython.pyimport(module_name, as: save_module_name)
  ScoutPython.send(save_module_name)
end

.import_method(module_name, method_name, as = nil) ⇒ Object 



40
41
42
43
44
 
# File 'lib/scout/python.rb', line 40

def self.import_method(module_name, method_name, as = nil)
  init_scout
  ScoutPython.pyfrom module_name, import: method_name
  ScoutPython.method(method_name)
end

.init_scout ⇒ Object 



13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/scout/python.rb', line 13

def self.init_scout
  if ! defined?(@@__init_scout_python) || ! @@__init_scout_python
    PyCall.init

    ScoutPython.process_paths
    res = ScoutPython.run_direct do
      Log.debug "Loading python 'scout' module into pycall ScoutPython module"
      pyimport("scout")
    end
    @@__init_scout_python = true

    at_exit do
      (Thread.list - [Thread.current]).each { |t| t.kill }
      (Thread.list - [Thread.current]).each { |t| t.join rescue nil }

      # GC while Python is still initialized so PyCall can safely acquire the GIL
      GC.start

      # (Optional) tiny no-op to ensure GIL path is healthy
      begin
        PyCall.builtins.object
      rescue => _
      end
    end
  end
end

.init_thread ⇒ Object 



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
 
# File 'lib/scout/python/run.rb', line 25

def self.init_thread
  if defined?(self.thread) && (self.thread && ! self.thread.alive?)
    Log.warn "Reloading ScoutPython thread"
    self.thread.join
    self.thread = nil
  end

  self.thread ||= Thread.new do
    require 'pycall'
    ScoutPython.init_scout
    ScoutPython.process_paths
    begin
      while block = QUEUE_IN.pop
        break if block == :stop
        res = 
          begin
            module_eval(&block)
          rescue Exception
            Log.exception $!
            raise $!
          end

        QUEUE_OUT.push res
      end
    rescue Exception
      Log.exception $!
      raise $!
    end
  end
end

.iterate(iterator, options = {}, &block) ⇒ Object 



101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
 
# File 'lib/scout/python.rb', line 101

def self.iterate(iterator, options = {}, &block)
  if ! iterator.respond_to?(:__next__)
    if iterator.respond_to?(:__iter__)
      iterator = iterator.__iter__
    else
      return iterate_index(iterator, options, &block)
    end
  end

  bar = options[:bar]

  case bar
  when TrueClass
    bar = Log::ProgressBar.new nil, :desc => "ScoutPython iterate"
  when String
    bar = Log::ProgressBar.new nil, :desc => bar
  end

  while true
    begin
      elem = iterator.__next__
      yield elem
      bar.tick if bar
    rescue PyCall::PyError
      if $!.type.to_s == "<class 'StopIteration'>"
        break
      else
        raise $!
      end
    rescue
      bar.error if bar
      raise $!
    end
  end

  Log::ProgressBar.remove_bar bar if bar
  nil
end

.iterate_index(elem, options = {}) ⇒ Object 



70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
 
# File 'lib/scout/python.rb', line 70

def self.iterate_index(elem, options = {})
  bar = options[:bar]

  len = PyCall.len(elem)
  case bar
  when TrueClass
    bar = Log::ProgressBar.new nil, :desc => "ScoutPython iterate"
  when String
    bar = Log::ProgressBar.new nil, :desc => bar
  end

  len.times do |i|
    begin
      yield elem[i]
      bar.tick if bar
    rescue PyCall::PyError
      if $!.type.to_s == "<class 'StopIteration'>"
        break
      else
        raise $!
      end
    rescue
      bar.error if bar
      raise $!
    end
  end

  Log::ProgressBar.remove_bar bar if bar
  nil
end

.list2ruby(list) ⇒ Object 



33
34
35
36
37
38
 
# File 'lib/scout/python/util.rb', line 33

def self.list2ruby(list)
  return list unless PyCall::List === list 
  list.collect do |e|
    list2ruby(e)
  end
end

.load_json(file) ⇒ Object 



84
85
86
 
# File 'lib/scout/python/script.rb', line 84

def self.load_json(file)
  JSON.load_file(file)
end

.load_pickle(file) ⇒ Object Also known as: load_result 



62
63
64
65
66
 
# File 'lib/scout/python/script.rb', line 62

def self.load_pickle(file)
  require 'python/pickle'
  Log.debug ("Loading pickle #{file}")
  Python::Pickle.load_file(file)
end

.load_script_variables(variables = {}) ⇒ Object 



30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
 
# File 'lib/scout/python/script.rb', line 30

def self.load_script_variables(variables = {})
  code = "# Variables\nimport scout\n"
  tmp_files = []
  variables.each do |name,value|
    case value
    when TSV
      tmp_file = TmpFile.tmp_file
      tmp_files << tmp_file
      Open.write(tmp_file, value.to_s)
      code << "#{name} = scout.tsv('#{tmp_file}')" << "\n"
    else
      code << "#{name} = #{ScoutPython.ruby2python(value)}" << "\n"
    end
  end

  [code, tmp_files]
end

.new_binding ⇒ Object 



149
150
151
 
# File 'lib/scout/python.rb', line 149

def self.new_binding
  Binding.new
end

.numpy2ruby(numpy) ⇒ Object 



40
41
42
 
# File 'lib/scout/python/util.rb', line 40

def self.numpy2ruby(numpy)
  list2ruby(numpy.tolist)
end

.obj2hash(obj) ⇒ Object 



44
45
46
47
48
49
50
 
# File 'lib/scout/python/util.rb', line 44

def self.obj2hash(obj)
  hash = {}
  ScoutPython.iterate obj.keys do |k|
    hash[k] = obj[k]
  end
  hash
end

.process_paths ⇒ Object 



17
18
19
20
21
22
23
24
 
# File 'lib/scout/python/paths.rb', line 17

def self.process_paths
  ScoutPython.run_direct 'sys' do
    ScoutPython.paths.each do |path|
      sys.path.append path
    end
    nil
  end
end

.py2ruby_a(array) ⇒ Object Also known as: to_a 



2
3
4
 
# File 'lib/scout/python/util.rb', line 2

def self.py2ruby_a(array)
  PyCall::List.(array).to_a
end

.ruby2python(object) ⇒ Object 



2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/scout/python/script.rb', line 2

def self.ruby2python(object)
  case object
  when Float::INFINITY
    "inf"
  when nil
    "None"
  when ":NA"
    "None"
  when Symbol
    "#{ object }"
  when String
    object = object.dup if Path === object
    object[0] == ":" ? object[1..-1] : "'#{ object }'"
  when Numeric
    object
  when TrueClass
    "True"
  when FalseClass
    "False"
  when Array
    "[#{object.collect{|e| ruby2python(e) } * ", "}]"
  when Hash
    "{" << object.collect{|k,v| [ruby2python(k.to_s), ruby2python(v)] * ":"} * ", " << "}"
  else
    raise "Type of object not known: #{ object.inspect }"
  end
end

.run ⇒ Object 



111
112
113
114
115
116
117
118
 
# File 'lib/scout/python/run.rb', line 111

def self.run(...)
  begin
    ScoutPython.init_scout
    run_simple(...)
  ensure
    GC.start
  end
end

.run_direct(mod = nil, imports = nil, &block) ⇒ Object 



76
77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/scout/python/run.rb', line 76

def self.run_direct(mod = nil, imports = nil, &block)
  if mod
    if Hash === imports
      pyimport mod, **imports
    elsif imports.nil?
      pyimport mod 
    else
      pyfrom mod, :import => imports
    end
  end 

  module_eval(&block)
end

.run_file(file, arg_str) ⇒ Object 



111
112
113
114
 
# File 'lib/scout/python/script.rb', line 111

def self.run_file(file, arg_str)
  path_env = ScoutPython.paths * ":"
  CMD.cmd("env PYTHONPATH=#{path_env} python '#{file}' #{arg_str}")
end

.run_in_thread(&block) ⇒ Object 



56
57
58
59
60
61
62
 
# File 'lib/scout/python/run.rb', line 56

def self.run_in_thread(&block)
  self.synchronize do
    init_thread
    QUEUE_IN.push block
    QUEUE_OUT.pop
  end
end

.run_log(mod = nil, imports = nil, severity = 0, severity_err = nil, &block) ⇒ Object 



120
121
122
123
124
 
# File 'lib/scout/python/run.rb', line 120

def self.run_log(mod = nil, imports = nil, severity = 0, severity_err = nil, &block)
  Log.trap_std("Python STDOUT", "Python STDERR", severity, severity_err) do
    run(mod, imports, &block)
  end
end

.run_log_stderr(mod = nil, imports = nil, severity = 0, &block) ⇒ Object 



126
127
128
129
130
 
# File 'lib/scout/python/run.rb', line 126

def self.run_log_stderr(mod = nil, imports = nil, severity = 0, &block)
  Log.trap_stderr("Python STDERR", severity) do
    run(mod, imports, &block)
  end
end

.run_simple(mod = nil, imports = nil, &block) ⇒ Object 



104
105
106
107
108
109
 
# File 'lib/scout/python/run.rb', line 104

def self.run_simple(mod = nil, imports = nil, &block)
  self.synchronize do
    ScoutPython.process_paths
    run_direct(mod, imports, &block)
  end
end

.run_threaded(mod = nil, imports = nil, &block) ⇒ Object 



90
91
92
93
94
95
96
97
98
99
100
101
102
 
# File 'lib/scout/python/run.rb', line 90

def self.run_threaded(mod = nil, imports = nil, &block)
  run_in_thread do
      if Hash === imports
        pyimport mod, **imports
      elsif imports.nil?
        pyimport mod 
      else
        pyfrom mod, :import => imports
      end
  end if mod

  run_in_thread(&block)
end

.save_script_result_json(file) ⇒ Object 



68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/scout/python/script.rb', line 68

def self.save_script_result_json(file)
  "\n# Save\ntry: result\nexcept NameError: result = None\nif result is not None:\nimport json\nfile = open('\#{file}', 'w', encoding='utf-8')\n# dump information to that file\nfile.write(json.dumps(result))\nfile.flush\nfile.close\n  EOF\nend\n"

.save_script_result_pickle(file) ⇒ Object Also known as: save_script_result 



48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/scout/python/script.rb', line 48

def self.save_script_result_pickle(file)
  "\n# Save\ntry: result\nexcept NameError: result = None\nif result is not None:\nimport pickle\nfile = open('\#{file}', 'wb')\n# dump information to that file\npickle.dump(result, file)\n  EOF\nend\n"

.script(text, variables = {}) ⇒ Object 



93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
 
# File 'lib/scout/python/script.rb', line 93

def self.script(text, variables = {})
  if variables.any?
    variable_definitions, tmp_files = load_script_variables(variables)
    text = variable_definitions + "\n# Script\n" + text
  end

  TmpFile.with_file do |tmp_file|
    text += save_script_result(tmp_file)
    Log.debug "Running python script:\n#{text.dup}"
    path_env = ScoutPython.paths * ":"
    CMD.cmd_log("env PYTHONPATH=#{path_env} python", {in: text})
    tmp_files.each{|file| Open.rm_rf file } if tmp_files
    if Open.exists?(tmp_file)
      load_result(tmp_file)
    end
  end
end

.stop_thread ⇒ Object 



64
65
66
67
68
69
70
71
72
73
74
 
# File 'lib/scout/python/run.rb', line 64

def self.stop_thread
  self.synchronize do
    if self.thread && self.thread.alive?
      QUEUE_IN.push :stop
      self.thread.join(2) || self.thread.kill
      GC.start
      PyCall.finalize if PyCall.respond_to?(:finalize)
    end
    self.thread = nil
  end
end

.synchronize(&block) ⇒ Object 



21
22
23
 
# File 'lib/scout/python/run.rb', line 21

def self.synchronize(&block)
  MUTEX.synchronize &block
end

.tsv2df(tsv) ⇒ Object 



10
11
12
13
14
15
16
17
 
# File 'lib/scout/python/util.rb', line 10

def self.tsv2df(tsv)
  df = nil
  ScoutPython.run_direct 'pandas' do
    df = pandas.DataFrame.new(tsv.values, columns: tsv.fields, index: tsv.keys)
    df.columns.name = tsv.key_field
  end
  df
end