Class: Twb::DataSource

Inherits:
Object
  • Object
show all
Includes:
TabTool
Defined in:
lib/twb/datasource.rb

Constant Summary collapse

@@hasher =
Digest::SHA256.new
@@connGNodeParamsJSON =
%q(
{ "csv"           : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "CSV"        },
  "excel"         : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "Excel"      },
  "dataengine"    : { "label" : ["dbname"  ], "id" : ["directory","filename"], "type" : "TDE"        },
  "msaccess"      : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "MS Access"  },
  "oracle"        : { "label" : ["server"  ], "id" : [            "server"  ], "type" : "Oracle"     },
  "postgres"      : { "label" : ["server"  ], "id" : [            "server"  ], "type" : "PostgreSQL" },
  "textscan"      : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "CSV / TSV"  },
  "excel-direct"  : { "label" : ["filename"], "id" : [            "filename"], "type" : "Excel"      },
  "salesforce"    : { "label" : ["server"  ], "id" : [            "server"  ], "type" : "Salesforce" }
}
)
@@cgNodeParams =
JSON.parse @@connGNodeParamsJSON

Instance Attribute Summary collapse

Attributes included from TabTool

#alerts, #docDir, #docfiles, #funcdoc, #licensed, #logfilename, #logger, #loglevel, #metrics, #properties, #ttdocdir, #type

Instance Method Summary collapse

Methods included from TabTool

#addDocFile, #alert, #closeDocFiles, #config, #docFile, #docFileMaxNameLen, #docfilesdoc, #docfilesdocto_s, #emit, #emitCSV, #finis, #hasConfig, #init, #initCSV, #initDocDir, #initLogger, #license=, #licensed?, #loadConfig

Constructor Details

#initialize(dataSourceNode, workbook) ⇒ DataSource

Returns a new instance of DataSource.


65
66
67
68
69
70
71
72
73
74
75
76
77
# File 'lib/twb/datasource.rb', line 65

def initialize dataSourceNode, workbook
  @node     = dataSourceNode
  @workbook = workbook 
  @name     = @node.attr('name')
  @caption  = @node.attr('caption')
  @uiname   = @caption.nil? ? @name : @caption
  # puts "DATASOURCE: #{@uiname}" 
  # processConnection
  # processFilters
  loadTableFields
  loadFieldUINames
  return self
end

Instance Attribute Details

#aliasesObject (readonly)

Returns the value of attribute aliases


58
59
60
# File 'lib/twb/datasource.rb', line 58

def aliases
  @aliases
end

#allFieldsObject (readonly)

Returns the value of attribute allFields


60
61
62
# File 'lib/twb/datasource.rb', line 60

def allFields
  @allFields
end

#attributesObject (readonly)

Returns the value of attribute attributes


45
46
47
# File 'lib/twb/datasource.rb', line 45

def attributes
  @attributes
end

#calculatedField(name) ⇒ Object (readonly)

Returns the value of attribute calculatedField


59
60
61
# File 'lib/twb/datasource.rb', line 59

def calculatedField
  @calculatedField
end

#calculatedFieldNamesObject (readonly)

Returns the value of attribute calculatedFieldNames


59
60
61
# File 'lib/twb/datasource.rb', line 59

def calculatedFieldNames
  @calculatedFieldNames
end

#calculatedFieldsObject (readonly)

Returns the value of attribute calculatedFields


59
60
61
# File 'lib/twb/datasource.rb', line 59

def calculatedFields
  @calculatedFields
end

#calculatedFieldsMapObject (readonly)

Returns the value of attribute calculatedFieldsMap


59
60
61
# File 'lib/twb/datasource.rb', line 59

def calculatedFieldsMap
  @calculatedFieldsMap
end

#captionObject (readonly)

Returns the value of attribute caption


44
45
46
# File 'lib/twb/datasource.rb', line 44

def caption
  @caption
end

#columnFieldsObject (readonly)

Returns the value of attribute columnFields


52
53
54
# File 'lib/twb/datasource.rb', line 52

def columnFields
  @columnFields
end

#connAttributesObject (readonly)

Returns the value of attribute connAttributes


47
48
49
# File 'lib/twb/datasource.rb', line 47

def connAttributes
  @connAttributes
end

#connectionObject (readonly)

Returns the value of attribute connection


47
48
49
# File 'lib/twb/datasource.rb', line 47

def connection
  @connection
end

#connectionsObject (readonly)

Returns the value of attribute connections


47
48
49
# File 'lib/twb/datasource.rb', line 47

def connections
  @connections
end

#connHashObject (readonly)

Returns the value of attribute connHash


47
48
49
# File 'lib/twb/datasource.rb', line 47

def connHash
  @connHash
end

#dbFieldsObject (readonly)

Returns the value of attribute dbFields


54
55
56
# File 'lib/twb/datasource.rb', line 54

def dbFields
  @dbFields
end

#dsclassObject (readonly)

Returns the value of attribute dsclass


46
47
48
# File 'lib/twb/datasource.rb', line 46

def dsclass
  @dsclass
end

#fieldUINamesObject (readonly)

Returns the value of attribute fieldUINames


57
58
59
# File 'lib/twb/datasource.rb', line 57

def fieldUINames
  @fieldUINames
end

#filtersObject (readonly)

Returns the value of attribute filters


62
63
64
# File 'lib/twb/datasource.rb', line 62

def filters
  @filters
end

#groupsObject (readonly)

Returns the value of attribute groups


61
62
63
# File 'lib/twb/datasource.rb', line 61

def groups
  @groups
end

#hasFieldObject (readonly)

Returns the value of attribute hasField


51
52
53
# File 'lib/twb/datasource.rb', line 51

def hasField
  @hasField
end

#isExtractObject (readonly)

Returns the value of attribute isExtract


46
47
48
# File 'lib/twb/datasource.rb', line 46

def isExtract
  @isExtract
end

#isPublishedObject (readonly)

Returns the value of attribute isPublished


48
49
50
# File 'lib/twb/datasource.rb', line 48

def isPublished
  @isPublished
end

#joinPairsObject (readonly)

Returns the value of attribute joinPairs


50
51
52
# File 'lib/twb/datasource.rb', line 50

def joinPairs
  @joinPairs
end

#localFieldObject (readonly)

Returns the value of attribute localField


51
52
53
# File 'lib/twb/datasource.rb', line 51

def localField
  @localField
end

#localFieldNamesObject (readonly)

Returns the value of attribute localFieldNames


51
52
53
# File 'lib/twb/datasource.rb', line 51

def localFieldNames
  @localFieldNames
end

#localFieldsObject (readonly)

Returns the value of attribute localFields


51
52
53
# File 'lib/twb/datasource.rb', line 51

def localFields
  @localFields
end

#mappedFieldsObject (readonly)

Returns the value of attribute mappedFields


55
56
57
# File 'lib/twb/datasource.rb', line 55

def mappedFields
  @mappedFields
end

#metadataFieldsObject (readonly)

Returns the value of attribute metadataFields


53
54
55
# File 'lib/twb/datasource.rb', line 53

def metadataFields
  @metadataFields
end

#nameObject (readonly)

Returns the value of attribute name


44
45
46
# File 'lib/twb/datasource.rb', line 44

def name
  @name
end

#nodeObject (readonly)

Returns the value of attribute node


63
64
65
# File 'lib/twb/datasource.rb', line 63

def node
  @node
end

#tableFieldsMapObject (readonly)

Returns the value of attribute tableFieldsMap


56
57
58
# File 'lib/twb/datasource.rb', line 56

def tableFieldsMap
  @tableFieldsMap
end

#tablesObject (readonly)

Returns the value of attribute tables


50
51
52
# File 'lib/twb/datasource.rb', line 50

def tables
  @tables
end

#uinameObject (readonly)

Returns the value of attribute uiname


44
45
46
# File 'lib/twb/datasource.rb', line 44

def uiname
  @uiname
end

#uuidObject (readonly)

Returns the value of attribute uuid


49
50
51
# File 'lib/twb/datasource.rb', line 49

def uuid
  @uuid
end

#workbookObject (readonly)

Returns the value of attribute workbook


43
44
45
# File 'lib/twb/datasource.rb', line 43

def workbook
  @workbook
end

Instance Method Details

#columnField(fieldName) ⇒ Object


238
239
240
241
# File 'lib/twb/datasource.rb', line 238

def columnField fieldName
  loadColumnFields if @columnFieldsMap.nil?
  @columnFieldsMap[ fieldName ]
end

#columnFieldsMapObject


248
249
250
251
# File 'lib/twb/datasource.rb', line 248

def columnFieldsMap
  loadColumnFields if @columnFieldsMap.nil?
  return @columnFieldsMap
end

#dbFieldsMapObject


398
399
400
# File 'lib/twb/datasource.rb', line 398

def dbFieldsMap
  @tableFieldsMap
end

#deAlias(fieldName, fAlias) ⇒ Object


324
325
326
327
328
329
# File 'lib/twb/datasource.rb', line 324

def deAlias fieldName, fAlias
  fldAliases = aliases[fieldName]
  return fAlias if fldAliases.nil?
  dbFieldValue = fldAliases.key(fAlias)
  dbFieldValue.nil? ? fAlias : dbFieldValue
end

#field(fieldName) ⇒ Object


459
460
461
# File 'lib/twb/datasource.rb', line 459

def field fieldName
  dbFieldsMap[fieldName]
end

#fieldAlias(fieldName, value) ⇒ Object


316
317
318
319
320
321
322
# File 'lib/twb/datasource.rb', line 316

def fieldAlias fieldName, value
  emit "fieldAlias:  #{fieldName.class}  -> #{value.class}" 
  loadAliases  if @aliases.nil?
  return value if @aliases.nil? || @aliases[fieldName].nil? # unless fieldHasAliases(fieldName)
  fldAlias = @aliases[fieldName][value]
  fldAlias.nil? ? value : fldAlias
end

#fieldAliases(fieldName) ⇒ Object


312
313
314
# File 'lib/twb/datasource.rb', line 312

def fieldAliases fieldName
  aliases[fieldName]
end

#fieldHasAliases(fieldName) ⇒ Object


308
309
310
# File 'lib/twb/datasource.rb', line 308

def fieldHasAliases fieldName
  !aliases[fieldName].nil?
end

#fieldTable(fieldName) ⇒ Object


463
464
465
466
467
# File 'lib/twb/datasource.rb', line 463

def fieldTable fieldName
  loadTableFields if @tableFieldsMap.nil?
  dbField = @tableFieldsMap[fieldName]
  return dbField.nil? ? nil : dbField.dbtable
end

#fieldUIName(fieldName) ⇒ Object


384
385
386
387
# File 'lib/twb/datasource.rb', line 384

def fieldUIName fieldName
  loadFieldUINames if @fieldUINames.nil?
  @fieldUINames[fieldName].nil? ? fieldName : @fieldUINames[fieldName]
end

#has_field?(fieldName) ⇒ Boolean

Returns:

  • (Boolean)

455
456
457
# File 'lib/twb/datasource.rb', line 455

def has_field? fieldName
  dbFieldsMap.has_key? fieldName
end

#idObject


97
98
99
# File 'lib/twb/datasource.rb', line 97

def id
    @id ||= @id = @name
end

#joinTreeObject


210
211
212
# File 'lib/twb/datasource.rb', line 210

def joinTree
  @joinTree ||= loadJoinTree
end

#loadAliasesObject


270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
# File 'lib/twb/datasource.rb', line 270

def loadAliases
  @aliases   = {}
  # puts $node.xpath('.//column/aliases/..').length
  cnt = 0
  @node.xpath('./column//aliases/..').each do |anode|
    # puts "anode:: #{anode}"
    # puts " path:: #{anode.path}"
    aliasMap = {}
    nameTech = anode.attribute('name').text.gsub(/^\[|\]$/,'')
    name     = fieldUIName nameTech
    if ':Measure Names'.eql? name
      # puts "processing Measure Names"
      anode.xpath('.//alias').each do |vnode|
        keyCode  = vnode.attribute('key').text.gsub(/^[#"%]|[#"%]$/,'')
        key      = Twb::CodedField.new(keyCode).name
        field    = fieldUIName key
        alia     = vnode.attribute('value').text
        # puts "  keyCode:: #{key}"
        # puts "  key    :: #{key}"
        # puts "  field  :: #{field}"
        # puts "  alias  :: #{alia}"
        aliasMap[field] = alia
      end
    else
      anode.xpath('.//alias').each do |vnode|
        key  = vnode.attribute('key').text.gsub(/^[#"]|[#"]$/,'')
        alia = vnode.attribute('value').text
        # puts " key :: #{key}"
        # puts "alias:: #{alia}"
        aliasMap[key] = alia
      end
    end
    @aliases[name] = aliasMap
  end
  emit "FIELD ALIASES: @aliases"
  return @aliases
end

#loadAllFieldsObject


449
450
451
452
453
# File 'lib/twb/datasource.rb', line 449

def loadAllFields
  @allFields = SortedSet.new
  @allFields << dbFields
  @allFields << calculatedFieldNames
end

#loadAttributes(node) ⇒ Object


87
88
89
90
91
92
93
94
95
# File 'lib/twb/datasource.rb', line 87

def loadAttributes node
  attributes = {}
  unless node.nil?
    node.attributes.each do |k,v|
      attributes[k] = v.text
    end
  end
  return attributes
end

#loadCalculatedFieldsObject


435
436
437
438
439
440
441
442
443
# File 'lib/twb/datasource.rb', line 435

def loadCalculatedFields
  @calculatedFieldsMap = {}
  cfnodes = @node.xpath("./column[calculation]")
  cfnodes.each do |node|
    calcField = Twb::CalculatedField.new node, self
    @calculatedFieldsMap[calcField.uiname] = calcField
  end
  return @calculatedFieldsMap
end

#loadColumnFieldsObject


253
254
255
256
257
258
259
260
261
262
263
264
# File 'lib/twb/datasource.rb', line 253

def loadColumnFields
  @columnFields    = Set.new
  @columnFieldsMap = {}
  nodes = @node.xpath('.//column')
  nodes.each do |n|
    field = Twb::ColumnField.new n, self
    @columnFields << field
    @columnFieldsMap[field.uiname] = field
    @columnFieldsMap[field.name]   = field
  end
  return @columnFields
end

#loadIsPublishedObject


176
177
178
# File 'lib/twb/datasource.rb', line 176

def loadIsPublished
  @isPublished = !node.at_xpath('./repository-location').nil?
end

#loadJoinPairsObject


199
200
201
202
203
204
205
206
207
208
# File 'lib/twb/datasource.rb', line 199

def loadJoinPairs
  @joinPairs = Set.new
  mainJoin   = @node.xpath("./connection/relation[@type='join']")
  clauses    = @node.xpath(".//relation[@type='join']/clause") 
  clauses.each do |clause|
    leafs = clause.xpath('.//expression[not(node())]')
    @joinPairs << [ pullTable(leafs[0]) , pullTable(leafs[1]) ]
  end
  return @joinPairs
end

#loadJoinTreeObject


214
215
216
217
218
219
220
221
222
223
224
225
226
227
# File 'lib/twb/datasource.rb', line 214

def loadJoinTree
  loadJoinPairs if @joinPairs.nil?
  # puts  "LJT::#{@uiname}::joinPairs:: #{@joinPairs.inspect}"
  # @joinPairs.each { |jp| # puts "JP::#{jp}" }
  @joinTree = JoinTree.new(@name)
  @joinPairs.each do |from,to|
    # puts  "from:#{from} -> to:#{to}"
    tableFrom = JoinTable.new(from)
    tableTo   = JoinTable.new(to)
    @joinTree.add(tableFrom, tableTo)
  end
  # puts  '---'
  return @joinTree
end

#loadLocalFieldsObject


335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
# File 'lib/twb/datasource.rb', line 335

def loadLocalFields
  @localFields = Set.new
  unless @connection.nil?  # Parameters has no connection node, & no local fields
    connClass    = @node.at_xpath('./connection').attribute('class').text
    fxpath       = case connClass
                   when 'dataengine' then './column'
                   when 'sqlserver'  then './column'
                   else                   './connection/relation/columns/column'
                   end
    nodes = @node.xpath(fxpath)
    nodes.each do |node|
      field = Twb::LocalField.new(node)
      @localFields << field
    end
  end
  return @localFields
end

#loadMetadataFieldsObject


357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
# File 'lib/twb/datasource.rb', line 357

def loadMetadataFields
  @metadataFields = Set.new
  unless @connection.nil?  # Parameters has no connection node, & no metadata fields
    # nodes = @node.xpath(".//metadata-record[@class='column']")
    # # note: there are other nodes "<metadata-record class='capability'>" whose nature is unclear
    # #       these nodes have no value for their <name node, so are not loaded
    nodes = @node.xpath("./connection//metadata-record[@class='column']")
    nodes.each do |node|
      field = Twb::MetadataField.new(node)
      field.source = :db
      @metadataFields << field
    end
    nodes = @node.xpath('./extract//metadata-record')
    nodes.each do |node|
      field = Twb::MetadataField.new(node)
      field.source = :extract
      @metadataFields << field
    end
  end
  return @metadataFields
end

#loadTableFieldsObject

fields are unique in the data source by UI name


470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
# File 'lib/twb/datasource.rb', line 470

def loadTableFields
  # puts  "DATA SOURCE FIELD TABLE LOAD"
  @tableFieldsMap = {}
  fieldNodes = @node.xpath('./connection/cols/map')
  fieldNodes.each do |fn|
    dbField = Twb::DbField.new(@uiname, fn, :map)
    @tableFieldsMap[dbField.uiname] = dbField
  end
  relTableNodes = @node.xpath('.//relation[@table]')
  relTableNodes.each do |relNode|
    table = relNode.attribute('name').text
    cols  = relNode.xpath('./columns/column')
    cols.each do |col|
      dbField = Twb::DbField.new(@uiname, col, :tableColumn, table)
      fldName = col.attribute('name')
      @tableFieldsMap[dbField.uiname] = dbField
    end
  end
end

#loadTablesObject


184
185
186
187
188
189
190
191
192
193
# File 'lib/twb/datasource.rb', line 184

def loadTables
  @tables = {}
  connections.each do |conn|
    nodes = conn.xpath(".//relation[@type='table']")
    nodes.each do |node|
      @tables[node.attr('name')] = node.attr('table')
    end
  end
  return @tables
end

#mappedFieldsMapObject


406
407
408
409
# File 'lib/twb/datasource.rb', line 406

def mappedFieldsMap
  loadTableFields if @tableFieldsMap.nil?
  return @tableFieldsMap
end

#Parameters?Boolean

Returns:

  • (Boolean)

234
235
236
# File 'lib/twb/datasource.rb', line 234

def Parameters?
  'Parameters'.eql? @name 
end

#processConnection(path) ⇒ Object


120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
# File 'lib/twb/datasource.rb', line 120

def processConnection path
    conns = @node.xpath(path)
    conns.each do |connNode|
      @connections << connNode
        # connClass = @dsclass
        # cpath     = connNode.path
        # connPath  = cpath.gsub(/\[[0-9]+\]/,'')
        # connPNum  = /(\d+)/.match(cpath)
        # # puts cpath, connPath
        # # puts "CPATH: #{cpath}"
        # conn.attributes.each do |name,value|
        #     # puts  "\n\t\t - %-15s -> %-s" % [name, value]
        #     $csvFile << [ $recCount += 1,
        #                   $twbName,  $dir,     $build,    $version,
        #                   $dsName,   $dstype,  $dsuiname, connClass, 
        #                   connPath,  connPNum,
        #                   name,      value.value
        #                 ]
        # end
    end
end

#processConnectionsObject


113
114
115
116
117
118
# File 'lib/twb/datasource.rb', line 113

def processConnections
  @connections = Array.new
  processConnection './/connection'
  processConnection './/named-connection'
  return @connections
end

#pullTable(xml) ⇒ Object


229
230
231
232
# File 'lib/twb/datasource.rb', line 229

def pullTable xml
    code =xml.attribute('op').text
    table = code.split('].[')[0][1..-1]
end

#setConnectionHashObject

Notes:

- TODO: need to determine which, if any, of the connection attributes should be
        included in the hash in order to identify it unambiguously - without
        local values that obscure the data source's 'real' identity
- attributes with value '' don't contribute to the hash

158
159
160
161
162
163
164
165
166
# File 'lib/twb/datasource.rb', line 158

def setConnectionHash
  dsAttributes = @node.xpath('./connection/@*')
  dsConnStr    = ''
  dsAttributes.each do |attr|
    dsConnStr += attr.text
  end
  @connHash = Digest::MD5.hexdigest(dsConnStr)
  @uuid = @connHash
end

#tableauVersionObject


101
102
103
# File 'lib/twb/datasource.rb', line 101

def tableauVersion tabVersion
  @tableauVersion = tabVersion
end

#tableFieldsObject


394
395
396
# File 'lib/twb/datasource.rb', line 394

def tableFields
  @tableFieldsMap.values
end

#updateTimeObject


379
380
381
382
# File 'lib/twb/datasource.rb', line 379

def updateTime
  attr = @node.xpath('.//@update-time').first
  @updateTime = attr.nil? ? nil : attr.value
end