Class: Twb::DataSource

Inherits:
Object
  • Object
show all
Includes:
TabTool
Defined in:
lib/twb/datasource.rb

Constant Summary collapse

@@hasher = 
Digest::SHA256.new
@@connGNodeParamsJSON = 
%q(
{ "csv"           : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "CSV"        },
  "excel"         : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "Excel"      },
  "dataengine"    : { "label" : ["dbname"  ], "id" : ["directory","filename"], "type" : "TDE"        },
  "msaccess"      : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "MS Access"  },
  "oracle"        : { "label" : ["server"  ], "id" : [            "server"  ], "type" : "Oracle"     },
  "postgres"      : { "label" : ["server"  ], "id" : [            "server"  ], "type" : "PostgreSQL" },
  "textscan"      : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "CSV / TSV"  },
  "excel-direct"  : { "label" : ["filename"], "id" : [            "filename"], "type" : "Excel"      },
  "salesforce"    : { "label" : ["server"  ], "id" : [            "server"  ], "type" : "Salesforce" }
}
)
@@cgNodeParams = 
JSON.parse @@connGNodeParamsJSON

Instance Attribute Summary collapse

Attributes included from TabTool

#alerts, #docDir, #docfiles, #funcdoc, #licensed, #logfilename, #logger, #loglevel, #metrics, #properties, #ttdocdir, #type

Instance Method Summary collapse

Methods included from TabTool

#addDocFile, #alert, #closeDocFiles, #config, #docFile, #docFileMaxNameLen, #docfilesdoc, #docfilesdocto_s, #emit, #emitCSV, #finis, #hasConfig, #init, #initCSV, #initDocDir, #initLogger, #license=, #licensed?, #loadConfig

Constructor Details

#initialize(dataSourceNode, workbook) ⇒ DataSource

Returns a new instance of DataSource.



65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/twb/datasource.rb', line 65

def initialize dataSourceNode, workbook
  @node     = dataSourceNode
  @workbook = workbook 
  @name     = @node.attr('name')
  @caption  = @node.attr('caption')
  @uiname   = @caption.nil? ? @name : @caption
  # puts "DATASOURCE: #{@uiname}" 
  # processConnection
  # processFilters
  loadTableFields
  loadFieldUINames
  return self
end

Instance Attribute Details

#aliases ⇒ Object (readonly)

Returns the value of attribute aliases.



58
59
60
 
# File 'lib/twb/datasource.rb', line 58

def aliases
  @aliases
end

#allFields ⇒ Object (readonly)

Returns the value of attribute allFields.



60
61
62
 
# File 'lib/twb/datasource.rb', line 60

def allFields
  @allFields
end

#attributes ⇒ Object (readonly)

Returns the value of attribute attributes.



45
46
47
 
# File 'lib/twb/datasource.rb', line 45

def attributes
  @attributes
end

#calculatedField(name) ⇒ Object (readonly)

Returns the value of attribute calculatedField.



59
60
61
 
# File 'lib/twb/datasource.rb', line 59

def calculatedField
  @calculatedField
end

#calculatedFieldNames ⇒ Object (readonly)

Returns the value of attribute calculatedFieldNames.



59
60
61
 
# File 'lib/twb/datasource.rb', line 59

def calculatedFieldNames
  @calculatedFieldNames
end

#calculatedFields ⇒ Object (readonly)

Returns the value of attribute calculatedFields.



59
60
61
 
# File 'lib/twb/datasource.rb', line 59

def calculatedFields
  @calculatedFields
end

#calculatedFieldsMap ⇒ Object (readonly)

Returns the value of attribute calculatedFieldsMap.



59
60
61
 
# File 'lib/twb/datasource.rb', line 59

def calculatedFieldsMap
  @calculatedFieldsMap
end

#caption ⇒ Object (readonly)

Returns the value of attribute caption.



44
45
46
 
# File 'lib/twb/datasource.rb', line 44

def caption
  @caption
end

#columnFields ⇒ Object (readonly)

Returns the value of attribute columnFields.



52
53
54
 
# File 'lib/twb/datasource.rb', line 52

def columnFields
  @columnFields
end

#connAttributes ⇒ Object (readonly)

Returns the value of attribute connAttributes.



47
48
49
 
# File 'lib/twb/datasource.rb', line 47

def connAttributes
  @connAttributes
end

#connection ⇒ Object (readonly)

Returns the value of attribute connection.



47
48
49
 
# File 'lib/twb/datasource.rb', line 47

def connection
  @connection
end

#connections ⇒ Object (readonly)

Returns the value of attribute connections.



47
48
49
 
# File 'lib/twb/datasource.rb', line 47

def connections
  @connections
end

#connHash ⇒ Object (readonly)

Returns the value of attribute connHash.



47
48
49
 
# File 'lib/twb/datasource.rb', line 47

def connHash
  @connHash
end

#dbFields ⇒ Object (readonly)

Returns the value of attribute dbFields.



54
55
56
 
# File 'lib/twb/datasource.rb', line 54

def dbFields
  @dbFields
end

#dsclass ⇒ Object (readonly)

Returns the value of attribute dsclass.



46
47
48
 
# File 'lib/twb/datasource.rb', line 46

def dsclass
  @dsclass
end

#fieldUINames ⇒ Object (readonly)

Returns the value of attribute fieldUINames.



57
58
59
 
# File 'lib/twb/datasource.rb', line 57

def fieldUINames
  @fieldUINames
end

#filters ⇒ Object (readonly)

Returns the value of attribute filters.



62
63
64
 
# File 'lib/twb/datasource.rb', line 62

def filters
  @filters
end

#groups ⇒ Object (readonly)

Returns the value of attribute groups.



61
62
63
 
# File 'lib/twb/datasource.rb', line 61

def groups
  @groups
end

#hasField ⇒ Object (readonly)

Returns the value of attribute hasField.



51
52
53
 
# File 'lib/twb/datasource.rb', line 51

def hasField
  @hasField
end

#isExtract ⇒ Object (readonly)

Returns the value of attribute isExtract.



46
47
48
 
# File 'lib/twb/datasource.rb', line 46

def isExtract
  @isExtract
end

#isPublished ⇒ Object (readonly)

Returns the value of attribute isPublished.



48
49
50
 
# File 'lib/twb/datasource.rb', line 48

def isPublished
  @isPublished
end

#joinPairs ⇒ Object (readonly)

Returns the value of attribute joinPairs.



50
51
52
 
# File 'lib/twb/datasource.rb', line 50

def joinPairs
  @joinPairs
end

#localField ⇒ Object (readonly)

Returns the value of attribute localField.



51
52
53
 
# File 'lib/twb/datasource.rb', line 51

def localField
  @localField
end

#localFieldNames ⇒ Object (readonly)

Returns the value of attribute localFieldNames.



51
52
53
 
# File 'lib/twb/datasource.rb', line 51

def localFieldNames
  @localFieldNames
end

#localFields ⇒ Object (readonly)

Returns the value of attribute localFields.



51
52
53
 
# File 'lib/twb/datasource.rb', line 51

def localFields
  @localFields
end

#mappedFields ⇒ Object (readonly)

Returns the value of attribute mappedFields.



55
56
57
 
# File 'lib/twb/datasource.rb', line 55

def mappedFields
  @mappedFields
end

#metadataFields ⇒ Object (readonly)

Returns the value of attribute metadataFields.



53
54
55
 
# File 'lib/twb/datasource.rb', line 53

def metadataFields
  @metadataFields
end

#name ⇒ Object (readonly)

Returns the value of attribute name.



44
45
46
 
# File 'lib/twb/datasource.rb', line 44

def name
  @name
end

#node ⇒ Object (readonly)

Returns the value of attribute node.



63
64
65
 
# File 'lib/twb/datasource.rb', line 63

def node
  @node
end

#tableFieldsMap ⇒ Object (readonly)

Returns the value of attribute tableFieldsMap.



56
57
58
 
# File 'lib/twb/datasource.rb', line 56

def tableFieldsMap
  @tableFieldsMap
end

#tables ⇒ Object (readonly)

Returns the value of attribute tables.



50
51
52
 
# File 'lib/twb/datasource.rb', line 50

def tables
  @tables
end

#uiname ⇒ Object (readonly)

Returns the value of attribute uiname.



44
45
46
 
# File 'lib/twb/datasource.rb', line 44

def uiname
  @uiname
end

#uuid ⇒ Object (readonly)

Returns the value of attribute uuid.



49
50
51
 
# File 'lib/twb/datasource.rb', line 49

def uuid
  @uuid
end

#workbook ⇒ Object (readonly)

Returns the value of attribute workbook.



43
44
45
 
# File 'lib/twb/datasource.rb', line 43

def workbook
  @workbook
end

Instance Method Details

#columnField(fieldName) ⇒ Object 



238
239
240
241
 
# File 'lib/twb/datasource.rb', line 238

def columnField fieldName
  loadColumnFields if @columnFieldsMap.nil?
  @columnFieldsMap[ fieldName ]
end

#columnFieldsMap ⇒ Object 



248
249
250
251
 
# File 'lib/twb/datasource.rb', line 248

def columnFieldsMap
  loadColumnFields if @columnFieldsMap.nil?
  return @columnFieldsMap
end

#dbFieldsMap ⇒ Object 



410
411
412
 
# File 'lib/twb/datasource.rb', line 410

def dbFieldsMap
  @tableFieldsMap
end

#deAlias(fieldName, fAlias) ⇒ Object 



336
337
338
339
340
341
 
# File 'lib/twb/datasource.rb', line 336

def deAlias fieldName, fAlias
  fldAliases = aliases[fieldName]
  return fAlias if fldAliases.nil?
  dbFieldValue = fldAliases.key(fAlias)
  dbFieldValue.nil? ? fAlias : dbFieldValue
end

#field(fieldName) ⇒ Object 



473
474
475
 
# File 'lib/twb/datasource.rb', line 473

def field fieldName
  dbFieldsMap[fieldName]
end

#fieldAlias(fieldName, value) ⇒ Object 



328
329
330
331
332
333
334
 
# File 'lib/twb/datasource.rb', line 328

def fieldAlias fieldName, value
  emit "fieldAlias:  #{fieldName.class}  -> #{value.class}" 
  loadAliases  if @aliases.nil?
  return value if @aliases.nil? || @aliases[fieldName].nil? # unless fieldHasAliases(fieldName)
  fldAlias = @aliases[fieldName][value]
  fldAlias.nil? ? value : fldAlias
end

#fieldAliases(fieldName) ⇒ Object 



324
325
326
 
# File 'lib/twb/datasource.rb', line 324

def fieldAliases fieldName
  aliases[fieldName]
end

#fieldHasAliases(fieldName) ⇒ Object 



320
321
322
 
# File 'lib/twb/datasource.rb', line 320

def fieldHasAliases fieldName
  !aliases[fieldName].nil?
end

#fieldTable(fieldName) ⇒ Object 



477
478
479
480
481
 
# File 'lib/twb/datasource.rb', line 477

def fieldTable fieldName
  loadTableFields if @tableFieldsMap.nil?
  dbField = @tableFieldsMap[fieldName]
  return dbField.nil? ? nil : dbField.dbtable
end

#fieldUIName(fieldName) ⇒ Object 



396
397
398
399
 
# File 'lib/twb/datasource.rb', line 396

def fieldUIName fieldName
  loadFieldUINames if @fieldUINames.nil?
  @fieldUINames[fieldName].nil? ? fieldName : @fieldUINames[fieldName]
end

#has_field?(fieldName) ⇒ Boolean

Returns:

  • (Boolean) 


469
470
471
 
# File 'lib/twb/datasource.rb', line 469

def has_field? fieldName
  dbFieldsMap.has_key? fieldName
end

#id ⇒ Object 



97
98
99
 
# File 'lib/twb/datasource.rb', line 97

def id
    @id ||= @id = @name
end

#joinTree ⇒ Object 



210
211
212
 
# File 'lib/twb/datasource.rb', line 210

def joinTree
  @joinTree ||= loadJoinTree
end

#loadAliases ⇒ Object 



282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
 
# File 'lib/twb/datasource.rb', line 282

def loadAliases
  @aliases   = {}
  # puts $node.xpath('.//column/aliases/..').length
  cnt = 0
  @node.xpath('./column//aliases/..').each do |anode|
    # puts "anode:: #{anode}"
    # puts " path:: #{anode.path}"
    aliasMap = {}
    nameTech = anode.attribute('name').text.gsub(/^\[|\]$/,'')
    name     = fieldUIName nameTech
    if ':Measure Names'.eql? name
      # puts "processing Measure Names"
      anode.xpath('.//alias').each do |vnode|
        keyCode  = vnode.attribute('key').text.gsub(/^[#"%]|[#"%]$/,'')
        key      = Twb::CodedField.new(keyCode).name
        field    = fieldUIName key
        alia     = vnode.attribute('value').text
        # puts "  keyCode:: #{key}"
        # puts "  key    :: #{key}"
        # puts "  field  :: #{field}"
        # puts "  alias  :: #{alia}"
        aliasMap[field] = alia
      end
    else
      anode.xpath('.//alias').each do |vnode|
        key  = vnode.attribute('key').text.gsub(/^[#"]|[#"]$/,'')
        alia = vnode.attribute('value').text
        # puts " key :: #{key}"
        # puts "alias:: #{alia}"
        aliasMap[key] = alia
      end
    end
    @aliases[name] = aliasMap
  end
  emit "FIELD ALIASES: @aliases"
  return @aliases
end

#loadAllFields ⇒ Object 



461
462
463
464
465
466
467
 
# File 'lib/twb/datasource.rb', line 461

def loadAllFields
  @allFields = SortedSet.new
  # dbFields.each { |f| @allFields << f }
  # calculatedFields.each { |f| @allFields << f }
  @allFields.merge dbFields
  @allFields.merge calculatedFields
end

#loadAttributes(node) ⇒ Object 



87
88
89
90
91
92
93
94
95
 
# File 'lib/twb/datasource.rb', line 87

def loadAttributes node
  attributes = {}
  unless node.nil?
    node.attributes.each do |k,v|
      attributes[k] = v.text
    end
  end
  return attributes
end

#loadCalculatedFields ⇒ Object 



447
448
449
450
451
452
453
454
455
 
# File 'lib/twb/datasource.rb', line 447

def loadCalculatedFields
  @calculatedFieldsMap = {}
  cfnodes = @node.xpath("./column[calculation]")
  cfnodes.each do |node|
    calcField = Twb::CalculatedField.new node, self
    @calculatedFieldsMap[calcField.uiname] = calcField
  end
  return @calculatedFieldsMap
end

#loadColumnFields ⇒ Object 



253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
 
# File 'lib/twb/datasource.rb', line 253

def loadColumnFields
  @columnFields    = Set.new
  @columnFieldsMap = {}
  nodes = @node.xpath('.//column')
  nodes.each do |n|
    # puts " "
    # puts @columnFieldsMap.keys.inspect
    name = n.attribute('name').text.sub(/^\[/,'').sub(/\]$/,'')
    loaded = @columnFieldsMap.has_key? name
    if loaded
       # puts "LOADED" 
       field = @columnFieldsMap[name]
       # puts "#{field.class}"
       field.loadAttributes(n)
    else 
       field = Twb::ColumnField.new n, self
       @columnFieldsMap[field.uiname] = field
       @columnFieldsMap[field.name]   = field
    end
    # puts "name:#{name}: \t nn:#{field.name}:\t loaded?:#{loaded}  node:#{n}:" 
  end
  @columnFields = @columnFieldsMap.values
  return @columnFields
end

#loadIsPublished ⇒ Object 



176
177
178
 
# File 'lib/twb/datasource.rb', line 176

def loadIsPublished
  @isPublished = !node.at_xpath('./repository-location').nil?
end

#loadJoinPairs ⇒ Object 



199
200
201
202
203
204
205
206
207
208
 
# File 'lib/twb/datasource.rb', line 199

def loadJoinPairs
  @joinPairs = Set.new
  mainJoin   = @node.xpath("./connection/relation[@type='join']")
  clauses    = @node.xpath(".//relation[@type='join']/clause") 
  clauses.each do |clause|
    leafs = clause.xpath('.//expression[not(node())]')
    @joinPairs << [ pullTable(leafs[0]) , pullTable(leafs[1]) ]
  end
  return @joinPairs
end

#loadJoinTree ⇒ Object 



214
215
216
217
218
219
220
221
222
223
224
225
226
227
 
# File 'lib/twb/datasource.rb', line 214

def loadJoinTree
  loadJoinPairs if @joinPairs.nil?
  # puts  "LJT::#{@uiname}::joinPairs:: #{@joinPairs.inspect}"
  # @joinPairs.each { |jp| # puts "JP::#{jp}" }
  @joinTree = JoinTree.new(@name)
  @joinPairs.each do |from,to|
    # puts  "from:#{from} -> to:#{to}"
    tableFrom = JoinTable.new(from)
    tableTo   = JoinTable.new(to)
    @joinTree.add(tableFrom, tableTo)
  end
  # puts  '---'
  return @joinTree
end

#loadLocalFields ⇒ Object 



347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
 
# File 'lib/twb/datasource.rb', line 347

def loadLocalFields
  @localFields = Set.new
  unless @connection.nil?  # Parameters has no connection node, & no local fields
    connClass    = @node.at_xpath('./connection').attribute('class').text
    fxpath       = case connClass
                   when 'dataengine' then './column'
                   when 'sqlserver'  then './column'
                   else                   './connection/relation/columns/column'
                   end
    nodes = @node.xpath(fxpath)
    nodes.each do |node|
      field = Twb::LocalField.new(node)
      @localFields << field
    end
  end
  return @localFields
end

#loadMetadataFields ⇒ Object 



369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
 
# File 'lib/twb/datasource.rb', line 369

def loadMetadataFields
  @metadataFields = Set.new
  unless @connection.nil?  # Parameters has no connection node, & no metadata fields
    # nodes = @node.xpath(".//metadata-record[@class='column']")
    # # note: there are other nodes "<metadata-record class='capability'>" whose nature is unclear
    # #       these nodes have no value for their <name node, so are not loaded
    nodes = @node.xpath("./connection//metadata-record[@class='column']")
    nodes.each do |node|
      field = Twb::MetadataField.new(node)
      field.source = :db
      @metadataFields << field
    end
    nodes = @node.xpath('./extract//metadata-record')
    nodes.each do |node|
      field = Twb::MetadataField.new(node)
      field.source = :extract
      @metadataFields << field
    end
  end
  return @metadataFields
end

#loadTableFields ⇒ Object

fields are unique in the data source by UI name



484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
 
# File 'lib/twb/datasource.rb', line 484

def loadTableFields
  # puts  "DATA SOURCE FIELD TABLE LOAD"
  @tableFieldsMap = {}
  fieldNodes = @node.xpath('./connection/cols/map')
  fieldNodes.each do |fn|
    dbField = Twb::DbField.new(@uiname, fn, :map)
    @tableFieldsMap[dbField.uiname] = dbField
  end
  relTableNodes = @node.xpath('.//relation[@table]')
  relTableNodes.each do |relNode|
    table = relNode.attribute('name').text
    cols  = relNode.xpath('./columns/column')
    cols.each do |col|
      dbField = Twb::DbField.new(@uiname, col, :tableColumn, table)
      fldName = col.attribute('name')
      @tableFieldsMap[dbField.uiname] = dbField
    end
  end
end

#loadTables ⇒ Object 



184
185
186
187
188
189
190
191
192
193
 
# File 'lib/twb/datasource.rb', line 184

def loadTables
  @tables = {}
  connections.each do |conn|
    nodes = conn.xpath(".//relation[@type='table']")
    nodes.each do |node|
      @tables[node.attr('name')] = node.attr('table')
    end
  end
  return @tables
end

#mappedFieldsMap ⇒ Object 



418
419
420
421
 
# File 'lib/twb/datasource.rb', line 418

def mappedFieldsMap
  loadTableFields if @tableFieldsMap.nil?
  return @tableFieldsMap
end

#Parameters? ⇒ Boolean

Returns:

  • (Boolean) 


234
235
236
 
# File 'lib/twb/datasource.rb', line 234

def Parameters?
  'Parameters'.eql? @name 
end

#processConnection(path) ⇒ Object 



120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
 
# File 'lib/twb/datasource.rb', line 120

def processConnection path
    conns = @node.xpath(path)
    conns.each do |connNode|
      @connections << connNode
        # connClass = @dsclass
        # cpath     = connNode.path
        # connPath  = cpath.gsub(/\[[0-9]+\]/,'')
        # connPNum  = /(\d+)/.match(cpath)
        # # puts cpath, connPath
        # # puts "CPATH: #{cpath}"
        # conn.attributes.each do |name,value|
        #     # puts  "\n\t\t - %-15s -> %-s" % [name, value]
        #     $csvFile << [ $recCount += 1,
        #                   $twbName,  $dir,     $build,    $version,
        #                   $dsName,   $dstype,  $dsuiname, connClass, 
        #                   connPath,  connPNum,
        #                   name,      value.value
        #                 ]
        # end
    end
end

#processConnections ⇒ Object 



113
114
115
116
117
118
 
# File 'lib/twb/datasource.rb', line 113

def processConnections
  @connections = Array.new
  processConnection './/connection'
  processConnection './/named-connection'
  return @connections
end

#pullTable(xml) ⇒ Object 



229
230
231
232
 
# File 'lib/twb/datasource.rb', line 229

def pullTable xml
    code =xml.attribute('op').text
    table = code.split('].[')[0][1..-1]
end

#setConnectionHash ⇒ Object

Notes:

- TODO: need to determine which, if any, of the connection attributes should be
        included in the hash in order to identify it unambiguously - without
        local values that obscure the data source's 'real' identity
- attributes with value '' don't contribute to the hash



158
159
160
161
162
163
164
165
166
 
# File 'lib/twb/datasource.rb', line 158

def setConnectionHash
  dsAttributes = @node.xpath('./connection/@*')
  dsConnStr    = ''
  dsAttributes.each do |attr|
    dsConnStr += attr.text
  end
  @connHash = Digest::MD5.hexdigest(dsConnStr)
  @uuid = @connHash
end

#tableauVersion ⇒ Object 



101
102
103
 
# File 'lib/twb/datasource.rb', line 101

def tableauVersion tabVersion
  @tableauVersion = tabVersion
end

#tableFields ⇒ Object 



406
407
408
 
# File 'lib/twb/datasource.rb', line 406

def tableFields
  @tableFieldsMap.values
end

#updateTime ⇒ Object 



391
392
393
394
 
# File 'lib/twb/datasource.rb', line 391

def updateTime
  attr = @node.xpath('.//@update-time').first
  @updateTime = attr.nil? ? nil : attr.value
end