Class: Twb::DataSource

Inherits:
Object
  • Object
show all
Includes:
TabTool
Defined in:
lib/twb/datasource.rb

Constant Summary collapse

@@hasher = 
Digest::SHA256.new
@@connGNodeParamsJSON = 
%q(
{ "csv"           : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "CSV"        },
  "excel"         : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "Excel"      },
  "dataengine"    : { "label" : ["dbname"  ], "id" : ["directory","filename"], "type" : "TDE"        },
  "msaccess"      : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "MS Access"  },
  "oracle"        : { "label" : ["server"  ], "id" : [            "server"  ], "type" : "Oracle"     },
  "postgres"      : { "label" : ["server"  ], "id" : [            "server"  ], "type" : "PostgreSQL" },
  "textscan"      : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "CSV / TSV"  },
  "excel-direct"  : { "label" : ["filename"], "id" : [            "filename"], "type" : "Excel"      },
  "salesforce"    : { "label" : ["server"  ], "id" : [            "server"  ], "type" : "Salesforce" }
}
)
@@cgNodeParams = 
JSON.parse @@connGNodeParamsJSON

Constants included from TabTool

TabTool::TTDOCDIR

Instance Attribute Summary collapse

Attributes included from TabTool

#alerts, #docfiles, #funcdoc, #licensed, #logfilename, #logger, #loglevel, #metrics, #properties, #ttdocdir, #type

Instance Method Summary collapse

Methods included from TabTool

#addDocFile, #alert, #docFile, #docFileMaxNameLen, #docfilesdoc, #emit, #emitCSV, #finis, #init, #initCSV, #initDocDir, #initLogger, #license=, #licensed?

Constructor Details

#initialize(dataSourceNode, workbook) ⇒ DataSource

Returns a new instance of DataSource.



62
63
64
65
66
67
68
69
70
71
72
73
74
 
# File 'lib/twb/datasource.rb', line 62

def initialize dataSourceNode, workbook
  @node     = dataSourceNode
  @workbook = workbook 
  @name     = @node.attr('name')
  @caption  = @node.attr('caption')
  @uiname   = @caption.nil? ? @name : @caption
  # puts "DATASOURCE: #{@uiname}" 

  processConnection
  processFilters
  loadTableFields
  loadFieldUINames
  return self
end

Instance Attribute Details

#aliases ⇒ Object (readonly)

Returns the value of attribute aliases.



55
56
57
 
# File 'lib/twb/datasource.rb', line 55

def aliases
  @aliases
end

#allFields ⇒ Object (readonly)

Returns the value of attribute allFields.



57
58
59
 
# File 'lib/twb/datasource.rb', line 57

def allFields
  @allFields
end

#calculatedField(name) ⇒ Object (readonly)

Returns the value of attribute calculatedField.



56
57
58
 
# File 'lib/twb/datasource.rb', line 56

def calculatedField
  @calculatedField
end

#calculatedFieldNames ⇒ Object (readonly)

Returns the value of attribute calculatedFieldNames.



56
57
58
 
# File 'lib/twb/datasource.rb', line 56

def calculatedFieldNames
  @calculatedFieldNames
end

#calculatedFieldNamesMap ⇒ Object (readonly)

Returns the value of attribute calculatedFieldNamesMap.



56
57
58
 
# File 'lib/twb/datasource.rb', line 56

def calculatedFieldNamesMap
  @calculatedFieldNamesMap
end

#calculatedFields ⇒ Object (readonly)

Returns the value of attribute calculatedFields.



56
57
58
 
# File 'lib/twb/datasource.rb', line 56

def calculatedFields
  @calculatedFields
end

#caption ⇒ Object (readonly)

Returns the value of attribute caption.



44
45
46
 
# File 'lib/twb/datasource.rb', line 44

def caption
  @caption
end

#columnFields ⇒ Object (readonly)

Returns the value of attribute columnFields.



49
50
51
 
# File 'lib/twb/datasource.rb', line 49

def columnFields
  @columnFields
end

#connection ⇒ Object (readonly)

Returns the value of attribute connection.



46
47
48
 
# File 'lib/twb/datasource.rb', line 46

def connection
  @connection
end

#connections ⇒ Object (readonly)

Returns the value of attribute connections.



46
47
48
 
# File 'lib/twb/datasource.rb', line 46

def connections
  @connections
end

#connHash ⇒ Object (readonly)

Returns the value of attribute connHash.



46
47
48
 
# File 'lib/twb/datasource.rb', line 46

def connHash
  @connHash
end

#dbFields ⇒ Object (readonly)

Returns the value of attribute dbFields.



51
52
53
 
# File 'lib/twb/datasource.rb', line 51

def dbFields
  @dbFields
end

#dsclass ⇒ Object (readonly)

Returns the value of attribute dsclass.



45
46
47
 
# File 'lib/twb/datasource.rb', line 45

def dsclass
  @dsclass
end

#fieldUINames ⇒ Object (readonly)

Returns the value of attribute fieldUINames.



54
55
56
 
# File 'lib/twb/datasource.rb', line 54

def fieldUINames
  @fieldUINames
end

#filters ⇒ Object (readonly)

Returns the value of attribute filters.



59
60
61
 
# File 'lib/twb/datasource.rb', line 59

def filters
  @filters
end

#groups ⇒ Object (readonly)

Returns the value of attribute groups.



58
59
60
 
# File 'lib/twb/datasource.rb', line 58

def groups
  @groups
end

#hasField ⇒ Object (readonly)

Returns the value of attribute hasField.



48
49
50
 
# File 'lib/twb/datasource.rb', line 48

def hasField
  @hasField
end

#isExtract ⇒ Object (readonly)

Returns the value of attribute isExtract.



45
46
47
 
# File 'lib/twb/datasource.rb', line 45

def isExtract
  @isExtract
end

#joinPairs ⇒ Object (readonly)

Returns the value of attribute joinPairs.



47
48
49
 
# File 'lib/twb/datasource.rb', line 47

def joinPairs
  @joinPairs
end

#localField ⇒ Object (readonly)

Returns the value of attribute localField.



48
49
50
 
# File 'lib/twb/datasource.rb', line 48

def localField
  @localField
end

#localFieldNames ⇒ Object (readonly)

Returns the value of attribute localFieldNames.



48
49
50
 
# File 'lib/twb/datasource.rb', line 48

def localFieldNames
  @localFieldNames
end

#localFields ⇒ Object (readonly)

Returns the value of attribute localFields.



48
49
50
 
# File 'lib/twb/datasource.rb', line 48

def localFields
  @localFields
end

#mappedFields ⇒ Object (readonly)

Returns the value of attribute mappedFields.



52
53
54
 
# File 'lib/twb/datasource.rb', line 52

def mappedFields
  @mappedFields
end

#metadataFields ⇒ Object (readonly)

Returns the value of attribute metadataFields.



50
51
52
 
# File 'lib/twb/datasource.rb', line 50

def metadataFields
  @metadataFields
end

#name ⇒ Object (readonly)

Returns the value of attribute name.



44
45
46
 
# File 'lib/twb/datasource.rb', line 44

def name
  @name
end

#node ⇒ Object (readonly)

Returns the value of attribute node.



60
61
62
 
# File 'lib/twb/datasource.rb', line 60

def node
  @node
end

#tableFieldsMap ⇒ Object (readonly)

Returns the value of attribute tableFieldsMap.



53
54
55
 
# File 'lib/twb/datasource.rb', line 53

def tableFieldsMap
  @tableFieldsMap
end

#tables ⇒ Object (readonly)

Returns the value of attribute tables.



47
48
49
 
# File 'lib/twb/datasource.rb', line 47

def tables
  @tables
end

#uiname ⇒ Object (readonly)

Returns the value of attribute uiname.



44
45
46
 
# File 'lib/twb/datasource.rb', line 44

def uiname
  @uiname
end

#uuid ⇒ Object (readonly)

Returns the value of attribute uuid.



46
47
48
 
# File 'lib/twb/datasource.rb', line 46

def uuid
  @uuid
end

#workbook ⇒ Object (readonly)

Returns the value of attribute workbook.



43
44
45
 
# File 'lib/twb/datasource.rb', line 43

def workbook
  @workbook
end

Instance Method Details

#calculatedFieldsMap ⇒ Object 



403
404
405
 
# File 'lib/twb/datasource.rb', line 403

def calculatedFieldsMap
  @calculatedFieldsMap ||= loadCalculatedFields
end

#columnFieldsMap ⇒ Object 



203
204
205
206
 
# File 'lib/twb/datasource.rb', line 203

def columnFieldsMap
  loadColumnFields if @columnFieldsMap.nil?
  return @columnFieldsMap
end

#dbFieldsMap ⇒ Object 



385
386
387
 
# File 'lib/twb/datasource.rb', line 385

def dbFieldsMap
  @tableFieldsMap
end

#deAlias(fieldName, fAlias) ⇒ Object 



277
278
279
280
281
282
 
# File 'lib/twb/datasource.rb', line 277

def deAlias fieldName, fAlias
  fldAliases = aliases[fieldName]
  return fAlias if fldAliases.nil?
  dbFieldValue = fldAliases.key(fAlias)
  dbFieldValue.nil? ? fAlias : dbFieldValue
end

#field(fieldName) ⇒ Object 



444
445
446
 
# File 'lib/twb/datasource.rb', line 444

def field fieldName
  dbFieldsMap[fieldName]
end

#fieldAlias(fieldName, value) ⇒ Object 



271
272
273
274
275
 
# File 'lib/twb/datasource.rb', line 271

def fieldAlias fieldName, value
  return value unless fieldHasAliases(fieldName)
  fldAlias = aliases[fieldName][value]
  fldAlias.nil? ? value : fldAlias
end

#fieldAliases(fieldName) ⇒ Object 



267
268
269
 
# File 'lib/twb/datasource.rb', line 267

def fieldAliases fieldName
  aliases[fieldName]
end

#fieldHasAliases(fieldName) ⇒ Object 



263
264
265
 
# File 'lib/twb/datasource.rb', line 263

def fieldHasAliases fieldName
  !aliases[fieldName].nil?
end

#fieldTable(fieldName) ⇒ Object 



448
449
450
451
452
 
# File 'lib/twb/datasource.rb', line 448

def fieldTable fieldName
  loadTableFields if @tableFieldsMap.nil?
  dbField = @tableFieldsMap[fieldName]
  return dbField.nil? ? nil : dbField.dbtable
end

#fieldUIName(fieldName) ⇒ Object 



337
338
339
340
 
# File 'lib/twb/datasource.rb', line 337

def fieldUIName fieldName
  loadFieldUINames if @fieldUINames.nil?
  @fieldUINames[fieldName]
end

#has_field?(fieldName) ⇒ Boolean

Returns:

  • (Boolean) 


440
441
442
 
# File 'lib/twb/datasource.rb', line 440

def has_field? fieldName
  dbFieldsMap.has_key? fieldName
end

#id ⇒ Object 



76
77
78
 
# File 'lib/twb/datasource.rb', line 76

def id
    @id ||= @id = @name
end

#joinTree ⇒ Object 



171
172
173
 
# File 'lib/twb/datasource.rb', line 171

def joinTree
  @joinTree ||= loadJoinTree
end

#loadAliases ⇒ Object 



225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
 
# File 'lib/twb/datasource.rb', line 225

def loadAliases
  @aliases   = {}
  # puts $node.xpath('.//column/aliases/..').length

  cnt = 0
  @node.xpath('./column//aliases/..').each do |anode|
    # puts "anode:: #{anode}"

    # puts " path:: #{anode.path}"

    aliasMap = {}
    nameTech = anode.attribute('name').text.gsub(/^\[|\]$/,'')
    name     = fieldUIName nameTech
    if ':Measure Names'.eql? name
      # puts "processing Measure Names"

      anode.xpath('.//alias').each do |vnode|
        keyCode  = vnode.attribute('key').text.gsub(/^[#"%]|[#"%]$/,'')
        key      = Twb::CodedField.new(keyCode).name
        field    = fieldUIName key
        alia     = vnode.attribute('value').text
        # puts "  keyCode:: #{key}"

        # puts "  key    :: #{key}"

        # puts "  field  :: #{field}"

        # puts "  alias  :: #{alia}"

        aliasMap[field] = alia
      end
    else
      anode.xpath('.//alias').each do |vnode|
        key  = vnode.attribute('key').text.gsub(/^[#"]|[#"]$/,'')
        alia = vnode.attribute('value').text
        # puts " key :: #{key}"

        # puts "alias:: #{alia}"

        aliasMap[key] = alia
      end
    end
    @aliases[name] = aliasMap
  end
  emit "FIELD ALIASES: @aliases"
  return @aliases
end

#loadCalculatedFields ⇒ Object 



422
423
424
425
426
427
428
429
430
 
# File 'lib/twb/datasource.rb', line 422

def loadCalculatedFields
  @calculatedFieldsMap = {}
  cfnodes = @node.xpath("./column[calculation]")
  cfnodes.each do |node|
    calcField = Twb::CalculatedField.new node, self
    @calculatedFieldsMap[calcField.uiname] = calcField
  end
  return @calculatedFieldsMap
end

#loadColumnFields ⇒ Object 



208
209
210
211
212
213
214
215
216
217
218
219
 
# File 'lib/twb/datasource.rb', line 208

def loadColumnFields
  @columnFields    = Set.new
  @columnFieldsMap = {}
  nodes = @node.xpath('./column')
  nodes.each do |n|
    field = Twb::ColumnField.new n, self
    @columnFields << field
    @columnFieldsMap[field.uiname] = field
    @columnFieldsMap[field.name]   = field
  end
  return @columnFields
end

#loadFieldUINames ⇒ Object 



347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
 
# File 'lib/twb/datasource.rb', line 347

def loadFieldUINames
  # puts 'loadFieldUINames'

  @fieldUINames = {}
  # puts 'metadataFields'

  metadataFields.each do |fld|
    # puts " - name:%-45s   |  uiname:%-45s   |  localName:%-45s " % [ "'#{fld.name}'", "'#{fld.uiname}'", "'#{fld.localName}'"]

    @fieldUINames[fld.uiname]    = fld.uiname
    @fieldUINames[fld.localName] = fld.uiname unless  fld.localName.nil?
    @fieldUINames[fld.name]      = fld.uiname unless  fld.name.nil?
  end
  # puts 'calculatedFields'

  calculatedFields.each do |fld| 
    # puts " - name:%-45s   |  uiname:%-45s " % [ "'#{fld.name}'", "'#{fld.uiname}'"]

    @fieldUINames[fld.name]   = fld.uiname
    @fieldUINames[fld.uiname] = fld.uiname
  end
  # puts 'localFields'

  localFields.each do |fld|
    # puts " - name:%-45s   |  uiname:%-45s " % [ "'#{fld.name}'", "'#{fld.uiname}'"]

    @fieldUINames[fld.name]   = fld.uiname
    @fieldUINames[fld.uiname] = fld.uiname
  end
  # puts "columnFields: #{columnFields.length}"

  columnFields.each do |fld|
    @fieldUINames[fld.name]   = fld.uiname
    @fieldUINames[fld.uiname] = fld.uiname
  end
  groups.each do |fld|
    @fieldUINames[fld.name]   = fld.uiname
    @fieldUINames[fld.uiname] = fld.uiname
  end
  return @fieldUINames
end

#loadGroups ⇒ Object 



479
480
481
482
483
484
485
486
487
 
# File 'lib/twb/datasource.rb', line 479

def loadGroups
  @groups = []
  groupNodes = @node.xpath('.//group')
  groupNodes.each do |groupNode|
    groupField = Twb::GroupField.new(groupNode)
    @groups << groupField
  end
  return @groups
end

#loadJoinPairs ⇒ Object 



160
161
162
163
164
165
166
167
168
169
 
# File 'lib/twb/datasource.rb', line 160

def loadJoinPairs
  @joinPairs = Set.new
  mainJoin   = @node.xpath("./connection/relation[@type='join']")
  clauses    = @node.xpath(".//relation[@type='join']/clause") 
  clauses.each do |clause|
    leafs = clause.xpath('.//expression[not(node())]')
    @joinPairs << [ pullTable(leafs[0]) , pullTable(leafs[1]) ]
  end
  return @joinPairs
end

#loadJoinTree ⇒ Object 



175
176
177
178
179
180
181
182
183
184
185
186
187
188
 
# File 'lib/twb/datasource.rb', line 175

def loadJoinTree
  loadJoinPairs if @joinPairs.nil?
  # puts  "LJT::#{@uiname}::joinPairs:: #{@joinPairs.inspect}"

  # @joinPairs.each { |jp| # puts "JP::#{jp}" }

  @joinTree = JoinTree.new(@name)
  @joinPairs.each do |from,to|
    # puts  "from:#{from} -> to:#{to}"

    tableFrom = JoinTable.new(from)
    tableTo   = JoinTable.new(to)
    @joinTree.add(tableFrom, tableTo)
  end
  # puts  '---'

  return @joinTree
end

#loadLocalFields ⇒ Object 



288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
 
# File 'lib/twb/datasource.rb', line 288

def loadLocalFields
  @localFields = Set.new
  unless @connection.nil?  # Parameters has no connection node, & no local fields

    connClass    = @node.at_xpath('./connection').attribute('class').text
    fxpath       = case connClass
                   when 'dataengine' then './column'
                   when 'sqlserver'  then './column'
                   else                   './connection/relation/columns/column'
                   end
    nodes = @node.xpath(fxpath)
    nodes.each do |node|
      field = Twb::LocalField.new(node)
      @localFields << field
    end
  end
  return @localFields
end

#loadMetadataFields ⇒ Object 



310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
 
# File 'lib/twb/datasource.rb', line 310

def loadMetadataFields
  @metadataFields = Set.new
  unless @connection.nil?  # Parameters has no connection node, & no metadata fields

    # nodes = @node.xpath(".//metadata-record[@class='column']")

    # # note: there are other nodes "<metadata-record class='capability'>" whose nature is unclear

    # #       these nodes have no value for their <name node, so are not loaded

    nodes = @node.xpath("./connection//metadata-record[@class='column']")
    nodes.each do |node|
      field = Twb::MetadataField.new(node)
      field.source = :db
      @metadataFields << field
    end
    nodes = @node.xpath('./extract//metadata-record')
    nodes.each do |node|
      field = Twb::MetadataField.new(node)
      field.source = :extract
      @metadataFields << field
    end
  end
  return @metadataFields
end

#loadTableFields ⇒ Object

fields are unique in the data source by UI name



455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
 
# File 'lib/twb/datasource.rb', line 455

def loadTableFields
  # puts  "DATA SOURCE FIELD TABLE LOAD"

  @tableFieldsMap = {}
  fieldNodes = @node.xpath('./connection/cols/map')
  fieldNodes.each do |fn|
    dbField = Twb::DbField.new(@uiname, fn, :map)
    @tableFieldsMap[dbField.uiname] = dbField
  end
  relTableNodes = @node.xpath('.//relation[@table]')
  relTableNodes.each do |relNode|
    table = relNode.attribute('name').text
    cols  = relNode.xpath('./columns/column')
    cols.each do |col|
      dbField = Twb::DbField.new(@uiname, col, :tableColumn, table)
      fldName = col.attribute('name')
      @tableFieldsMap[dbField.uiname] = dbField
    end
  end
end

#loadTables(connection) ⇒ Object 



148
149
150
151
152
153
154
 
# File 'lib/twb/datasource.rb', line 148

def loadTables connection
  @tables = {}
  nodes = connection.xpath(".//relation[@type='table']")
  nodes.each do |node|
    @tables[node.attr('name')] = node.attr('table')
  end
end

#mappedFieldsMap ⇒ Object 



393
394
395
396
 
# File 'lib/twb/datasource.rb', line 393

def mappedFieldsMap
  loadTableFields if @tableFieldsMap.nil?
  return @tableFieldsMap
end

#Parameters? ⇒ Boolean

Returns:

  • (Boolean) 


195
196
197
 
# File 'lib/twb/datasource.rb', line 195

def Parameters?
  'Parameters'.eql? @name 
end

#processConnection ⇒ Object 



97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/twb/datasource.rb', line 97

def processConnection path
    conns = @node.xpath(path)
    conns.each do |connNode|
        connClass = @dsclass
        cpath     = conn.path
        connPath  = cpath.gsub(/\[[0-9]+\]/,'')
        connPNum  = /(\d+)/.match(cpath)
        # puts cpath, connPath

        # puts "CPATH: #{cpath}"

        conn.attributes.each do |name,value|
            # puts  "\n\t\t - %-15s -> %-s" % [name, value]

            $csvFile << [ $recCount += 1,
                          $twbName,  $dir,     $build,    $version,
                          $dsName,   $dstype,  $dsuiname, connClass, 
                          connPath,  connPNum,
                          name,      value.value
                        ]
        end
    end
end

#processConnections ⇒ Object 



92
93
94
95
 
# File 'lib/twb/datasource.rb', line 92

def processConnections
  processConnection ds, './/connection'
  processConnection ds, './/named-connection'
end

#processFilters ⇒ Object 



489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
 
# File 'lib/twb/datasource.rb', line 489

def processFilters
  if @filters.nil?
    @filters = {}
    fnodes = @node.xpath('./filter')
    fnodes.each do |fn|
      columnN = fn.attribute('column')
      unless columnN.nil?
        memberNodes = fn.xpath('.//groupfilter/@member')
        unless memberNodes.nil?
          members = []
          memberNodes.each do |m|
            members.push m.text
          end
          @filters[columnN.text] = members
        end
      end 
    end
  end
end

#pullTable(xml) ⇒ Object 



190
191
192
193
 
# File 'lib/twb/datasource.rb', line 190

def pullTable xml
    code =xml.attribute('op').text
    table = code.split('].[')[0][1..-1]
end

#setConnectionHash ⇒ Object

Notes:

- TODO: need to determine which, if any, of the connection attributes should be
        included in the hash in order to identify it unambiguously - without
        local values that obscure the data source's 'real' identity
- attributes with value '' don't contribute to the hash



134
135
136
137
138
139
140
141
142
 
# File 'lib/twb/datasource.rb', line 134

def setConnectionHash
  dsAttributes = @node.xpath('./connection/@*')
  dsConnStr    = ''
  dsAttributes.each do |attr|
    dsConnStr += attr.text
  end
  @connHash = Digest::MD5.hexdigest(dsConnStr)
  @uuid = @connHash
end

#tableauVersion ⇒ Object 



80
81
82
 
# File 'lib/twb/datasource.rb', line 80

def tableauVersion tabVersion
  @tableauVersion = tabVersion
end

#tableFields ⇒ Object 



381
382
383
 
# File 'lib/twb/datasource.rb', line 381

def tableFields
  @tableFieldsMap.values
end

#updateTime ⇒ Object 



332
333
334
335
 
# File 'lib/twb/datasource.rb', line 332

def updateTime
  attr = @node.xpath('.//@update-time').first
  @updateTime = attr.nil? ? nil : attr.value
end