Class: PuppetDBQuery::MongoDB

Inherits:
Object
  • Object
show all
Includes:
Logging
Defined in:
lib/puppetdb_query/mongodb.rb

Overview

access nodes and their facts from mongo database rubocop:disable Metrics/ClassLength

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Logging

included, #logger, #logger=

Constructor Details

#initialize(connection, nodes = :nodes, node_properties = :node_properties, meta = :meta) ⇒ MongoDB

initialize access to mongodb

You might want to adjust the logging level, for example:

::Mongo::Logger.logger.level = logger.level

Parameters:

  • connection —

    mongodb connection, should already be switched to correct database

  • nodes (defaults to: :nodes) —

    symbol for collection that contains nodes with their facts

  • node_properties (defaults to: :node_properties) —

    symbol for collection for nodes with their update timestamps

  • meta (defaults to: :meta) —

    symbol for collection with update metadata



39
40
41
42
43
44
 
# File 'lib/puppetdb_query/mongodb.rb', line 39

def initialize(connection, nodes = :nodes, node_properties = :node_properties, meta = :meta)
  @connection = connection
  @nodes_collection = nodes
  @node_properties_collection = node_properties
  @meta_collection = meta
end

Instance Attribute Details

#connection ⇒ Object (readonly)

Returns the value of attribute connection.



24
25
26
 
# File 'lib/puppetdb_query/mongodb.rb', line 24

def connection
  @connection
end

#meta_collection ⇒ Object (readonly)

Returns the value of attribute meta_collection.



27
28
29
 
# File 'lib/puppetdb_query/mongodb.rb', line 27

def meta_collection
  @meta_collection
end

#node_properties_collection ⇒ Object (readonly)

Returns the value of attribute node_properties_collection.



26
27
28
 
# File 'lib/puppetdb_query/mongodb.rb', line 26

def node_properties_collection
  @node_properties_collection
end

#node_properties_update_timestamp ⇒ Object (readonly)

Returns the value of attribute node_properties_update_timestamp.



28
29
30
 
# File 'lib/puppetdb_query/mongodb.rb', line 28

def node_properties_update_timestamp
  @node_properties_update_timestamp
end

#nodes_collection ⇒ Object (readonly)

Returns the value of attribute nodes_collection.



25
26
27
 
# File 'lib/puppetdb_query/mongodb.rb', line 25

def nodes_collection
  @nodes_collection
end

Instance Method Details

#all_nodes ⇒ Object

get all node names



58
59
60
61
 
# File 'lib/puppetdb_query/mongodb.rb', line 58

def all_nodes
  collection = connection[nodes_collection]
  collection.find.batch_size(999).projection(_id: 1).map { |k| k[:_id] }
end

#facts(facts = []) ⇒ Object

get all nodes and their facts

Parameters:

  • facts (Array<String>) (defaults to: []) —

    get these facts in the result, eg [‘fqdn’], empty for all



146
147
148
149
150
151
152
153
154
155
 
# File 'lib/puppetdb_query/mongodb.rb', line 146

def facts(facts = [])
  fields = Hash[facts.collect { |fact| [fact.to_sym, 1] }]
  collection = connection[nodes_collection]
  result = {}
  collection.find.batch_size(999).projection(fields).each do |values|
    id = values.delete('_id')
    result[id] = values
  end
  result
end

#meta ⇒ Object

get meta informations about updates



158
159
160
161
162
163
 
# File 'lib/puppetdb_query/mongodb.rb', line 158

def meta
  collection = connection[meta_collection]
  result = collection.find.first
  result.delete(:_id)
  result
end

#meta_fact_update(method, ts_begin, ts_end) ⇒ Object

update or insert timestamps for given fact update method



225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
 
# File 'lib/puppetdb_query/mongodb.rb', line 225

def meta_fact_update(method, ts_begin, ts_end)
  connection[meta_collection].find_one_and_update(
    {},
    {
      '$set' => {
        last_fact_update: {
          ts_begin: ts_begin.iso8601,
          ts_end:   ts_end.iso8601,
          method:   method
        },
        method => {
          ts_begin: ts_begin.iso8601,
          ts_end:   ts_end.iso8601
        }
      }
    },
    { upsert: true }
  )
end

#meta_node_properties_update(ts_begin, ts_end) ⇒ Object

update or insert timestamps for node_properties_update



246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
 
# File 'lib/puppetdb_query/mongodb.rb', line 246

def meta_node_properties_update(ts_begin, ts_end)
  connection[meta_collection].find_one_and_update(
    {},
    {
      '$set' => {
        last_node_properties_update: {
          ts_begin: ts_begin.iso8601,
          ts_end:   ts_end.iso8601
        }
      }
    },
    { upsert: true }
  )
  @node_properties_update_timestamp = ts_begin
end

#node_delete(node) ⇒ Object

delete node data for given node name

Parameters:

  • node (String) —

    node name



201
202
203
 
# File 'lib/puppetdb_query/mongodb.rb', line 201

def node_delete(node)
  connection[nodes_collection].find(_id: node).delete_one
end

#node_properties ⇒ Object

get all nodes and their update dates



47
48
49
50
51
52
53
54
55
 
# File 'lib/puppetdb_query/mongodb.rb', line 47

def node_properties
  collection = connection[node_properties_collection]
  result = {}
  collection.find.batch_size(999).each do |values|
    id = values.delete('_id')
    result[id] = values
  end
  result
end

#node_properties_update(new_node_properties) ⇒ Object

update node properties



206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
 
# File 'lib/puppetdb_query/mongodb.rb', line 206

def node_properties_update(new_node_properties)
  collection = connection[node_properties_collection]
  old_names = collection.find.batch_size(999).projection(_id: 1).map { |k| k[:_id] }
  delete = old_names - new_node_properties.keys
  data = new_node_properties.map do |k, v|
    {
      replace_one:
      {
        filter: { _id: k },
        replacement: v,
        upsert: true
      }
    }
  end
  collection.bulk_write(data)
  collection.delete_many(_id: { '$in' => delete })
end

#node_update(node, facts) ⇒ Object

update or insert facts for given node name

Parameters:

  • node (String) —

    node name

  • facts (Hash) —

    facts for the node



169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
 
# File 'lib/puppetdb_query/mongodb.rb', line 169

def node_update(node, facts)
  logger.debug "  updating #{node}"
  connection[nodes_collection].find(_id: node).replace_one(facts,
    upsert: true,
    bypass_document_validation: true,
    check_keys: false,
    validating_keys: false)
rescue ::Mongo::Error::OperationFailure => e
  logger.warn "  updating #{node} failed with: #{e.message}"
  # mongodb doesn't support keys with a dot
  # see https://docs.mongodb.com/manual/reference/limits/#Restrictions-on-Field-Names
  # as a dirty workaround we delete the document and insert it ;-)
  # The dotted field .. in .. is not valid for storage. (57)
  # .. is an illegal key in MongoDB. Keys may not start with '$' or contain a '.'.
  # (BSON::String::IllegalKey)
  raise e unless e.message =~ /The dotted field / || e.message =~ /is an illegal key/
  logger.warn "    we transform the dots into underline characters"
  begin
    facts = Hash[facts.map { |k, v| [k.tr('.', '_'), v] }]
    connection[nodes_collection].find(_id: node).replace_one(facts,
      upsert: true,
      bypass_document_validation: true,
      check_keys: false,
      validating_keys: false)
  rescue
    logger.error "  inserting node #{node} failed again with: #{e.message}"
  end
end

#query_facts(query, facts = []) ⇒ Object

get nodes and their facts that fulfill given mongodb query

Parameters:

  • query —

    mongodb query

  • facts (Array<String>) (defaults to: []) —

    get these facts in the result, eg [‘fqdn’], empty for all



75
76
77
78
79
80
81
82
83
84
 
# File 'lib/puppetdb_query/mongodb.rb', line 75

def query_facts(query, facts = [])
  fields = Hash[facts.collect { |fact| [fact.to_sym, 1] }]
  collection = connection[nodes_collection]
  result = {}
  collection.find(query).batch_size(999).projection(fields).each do |values|
    id = values.delete('_id')
    result[id] = values
  end
  result
end

#query_facts_exist(query, facts = []) ⇒ Object

get nodes and their facts that fulfill given mongodb query and have at least one value for one the given fact names

Parameters:

  • query —

    mongodb query

  • facts (Array<String>) (defaults to: []) —

    get these facts in the result, eg [‘fqdn’], empty for all



91
92
93
94
95
96
97
98
99
 
# File 'lib/puppetdb_query/mongodb.rb', line 91

def query_facts_exist(query, facts = [])
  result = query_facts(query, facts)
  unless facts.empty?
    result.keep_if do |_k, v|
      facts.any? { |f| !v[f].nil? }
    end
  end
  result
end

#query_nodes(query) ⇒ Object

get node names that fulfill given mongodb query

Parameters:

  • query —

    mongodb query



66
67
68
69
 
# File 'lib/puppetdb_query/mongodb.rb', line 66

def query_nodes(query)
  collection = connection[nodes_collection]
  collection.find(query).batch_size(999).projection(_id: 1).map { |k| k[:_id] }
end

#search_facts(query, pattern, facts = [], facts_found = [], check_names = false) ⇒ Object

get nodes and their facts for a pattern

Parameters:

  • query —

    mongodb query

  • pattern (RegExp) —

    search for

  • facts (Array<String>) (defaults to: []) —

    get these facts in the result, eg [‘fqdn’], empty for all

  • facts_found (Array<String>) (defaults to: []) —

    fact names are added to this array

  • check_names (Boolean) (defaults to: false) —

    also search fact names



108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
 
# File 'lib/puppetdb_query/mongodb.rb', line 108

def search_facts(query, pattern, facts = [], facts_found = [], check_names = false)
  collection = connection[nodes_collection]
  result = {}
  collection.find(query).batch_size(999).each do |values|
    id = values.delete('_id')
    found = {}
    values.each do |k, v|
      if v =~ pattern
        found[k] = v
      elsif check_names && k =~ pattern
        found[k] = v
      end
    end
    next if found.empty?
    facts_found.concat(found.keys).uniq!
    facts.each do |f|
      found[f] = values[f]
    end
    result[id] = found
  end
  result
end

#single_node_facts(node, facts) ⇒ Object

get facts for given node name

Parameters:

  • node (String) —

    node name

  • facts (Array<String>) —

    get these facts in the result, eg [‘fqdn’], empty for all



135
136
137
138
139
140
141
 
# File 'lib/puppetdb_query/mongodb.rb', line 135

def single_node_facts(node, facts)
  fields = Hash[facts.collect { |fact| [fact.to_sym, 1] }]
  collection = connection[nodes_collection]
  result = collection.find(_id: node).limit(1).batch_size(1).projection(fields).to_a.first
  result.delete("_id") if result
  result
end