Class: Ole::Storage

Inherits:
Object
  • Object
show all
Defined in:
lib/ole/storage/base.rb,
lib/ole/storage/version.rb,
lib/ole/storage/meta_data.rb,
lib/ole/storage/file_system.rb

Overview

:nodoc:

Defined Under Namespace

Classes: AllocationTable, DirClass, Dirent, FileClass, FormatError, Header, MetaData, RangesIOMigrateable, RangesIOResizeable

Constant Summary collapse

VERSION = 
'1.2.13'

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(arg, mode = nil, params = {}) ⇒ Storage

arg should be either a filename, or an IO object, and needs to be seekable. mode is optional, and should be a regular mode string.



40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
 
# File 'lib/ole/storage/base.rb', line 40

def initialize arg, mode=nil, params={}
  params, mode = mode, nil if Hash === mode
  params = {:update_timestamps => true}.merge(params)
  @params = params
  
  # get the io object
  @close_parent, @io = if String === arg
    mode ||= 'rb'
    [true, open(arg, mode)]
  else
    raise ArgumentError, 'unable to specify mode string with io object' if mode
    [false, arg]
  end
  # force encoding, to avoid picking up source encoding with StringIO or files in text mode
  @io.set_encoding Encoding::ASCII_8BIT if @io.respond_to?(:set_encoding)
  # do we have this file opened for writing? use mode when provided,
  # otherwise try no-op methods which will raise if read-only
  @writeable = begin
    if mode
      IOMode.new(mode).writeable?
    else
      # works on mri 1.8 & jruby
      @io.flush
      begin
        # works on mri 1.9 & rubinius, throws EBADF on windows
        @io.write_nonblock('') if @io.respond_to?(:write_nonblock)
      rescue Errno::EBADF
        # for windows
        @io.syswrite('');
      end
      true
    end
  rescue IOError
    false
  end
  # silence undefined warning in clear
  @sb_file = nil
  # if the io object has data, we should load it, otherwise start afresh
  # this should be based on the mode string rather.
  @io.size > 0 ? load : clear
end

Instance Attribute Details

#bbat ⇒ Object (readonly)

Low level internals, you probably shouldn’t need to mess with these



36
37
38
 
# File 'lib/ole/storage/base.rb', line 36

def bbat
  @bbat
end

#close_parent ⇒ Object (readonly)

The underlying io object to/from which the ole object is serialized, whether we should close it, and whether it is writeable



34
35
36
 
# File 'lib/ole/storage/base.rb', line 34

def close_parent
  @close_parent
end

#dirents ⇒ Object (readonly)

The tree structure in its original flattened form. only valid after #load, or #flush.



31
32
33
 
# File 'lib/ole/storage/base.rb', line 31

def dirents
  @dirents
end

#header ⇒ Object (readonly)

Low level internals, you probably shouldn’t need to mess with these



36
37
38
 
# File 'lib/ole/storage/base.rb', line 36

def header
  @header
end

#io ⇒ Object (readonly)

The underlying io object to/from which the ole object is serialized, whether we should close it, and whether it is writeable



34
35
36
 
# File 'lib/ole/storage/base.rb', line 34

def io
  @io
end

#params ⇒ Object (readonly)

options used at creation time



27
28
29
 
# File 'lib/ole/storage/base.rb', line 27

def params
  @params
end

#root ⇒ Object (readonly)

The top of the ole tree structure



29
30
31
 
# File 'lib/ole/storage/base.rb', line 29

def root
  @root
end

#sb_file ⇒ Object (readonly)

Low level internals, you probably shouldn’t need to mess with these



36
37
38
 
# File 'lib/ole/storage/base.rb', line 36

def sb_file
  @sb_file
end

#sbat ⇒ Object (readonly)

Low level internals, you probably shouldn’t need to mess with these



36
37
38
 
# File 'lib/ole/storage/base.rb', line 36

def sbat
  @sbat
end

#writeable ⇒ Object (readonly)

The underlying io object to/from which the ole object is serialized, whether we should close it, and whether it is writeable



34
35
36
 
# File 'lib/ole/storage/base.rb', line 34

def writeable
  @writeable
end

Class Method Details

.open(arg, mode = nil, params = {}) ⇒ Object

somewhat similar to File.open, the open class method allows a block form where the Ole::Storage object is automatically closed on completion of the block.



84
85
86
87
88
89
90
91
92
 
# File 'lib/ole/storage/base.rb', line 84

def self.open arg, mode=nil, params={}
  ole = new arg, mode, params
  if block_given?
    begin   yield ole
    ensure; ole.close
    end
  else ole
  end
end

Instance Method Details

#bat_for_size(size) ⇒ Object 



333
334
335
336
 
# File 'lib/ole/storage/base.rb', line 333

def bat_for_size size
  # note >=, not > previously.
  size >= @header.threshold ? @bbat : @sbat
end

#clear ⇒ Object 



295
296
297
298
299
300
301
302
303
304
305
306
307
308
 
# File 'lib/ole/storage/base.rb', line 295

def clear
  # initialize to equivalent of loading an empty ole document.
  Log.warn 'creating new ole storage object on non-writable io' unless @writeable
  @header = Header.new
  @bbat = AllocationTable::Big.new self
  @root = Dirent.new self, :type => :root, :name => 'Root Entry'
  @dirents = [@root]
  @root.idx = 0
  @sb_file.close if @sb_file
  @sb_file = RangesIOResizeable.new @bbat, :first_block => AllocationTable::EOC
  @sbat = AllocationTable::Small.new self
  # throw everything else the hell away
  @io.truncate 0
end

#close ⇒ Object 



164
165
166
167
168
 
# File 'lib/ole/storage/base.rb', line 164

def close
  @sb_file.close
  flush if @writeable
  @io.close if @close_parent
end

#dir ⇒ Object 



40
41
42
 
# File 'lib/ole/storage/file_system.rb', line 40

def dir
  @dir ||= DirClass.new self
end

#dirent_from_path(path) ⇒ Object

tries to get a dirent for path. return nil if it doesn’t exist (change it)



46
47
48
49
50
51
52
53
54
55
56
 
# File 'lib/ole/storage/file_system.rb', line 46

def dirent_from_path path
  dirent = @root
  path = file.expand_path(path).split('/')
  until path.empty?
    part = path.shift
    next if part.empty?
    return nil if dirent.file?
    return nil unless dirent = dirent/part
  end
  dirent
end

#file ⇒ Object 



36
37
38
 
# File 'lib/ole/storage/file_system.rb', line 36

def file
  @file ||= FileClass.new self
end

#flush ⇒ Object

the flush method is the main “save” method. all file contents are always written directly to the file by the RangesIO objects, all this method does is write out all the file meta data - dirents, allocation tables, file header etc.

maybe add an option to zero the padding, and any remaining avail blocks in the allocation table.

TODO: long and overly complex. simplify and test better. eg, perhaps move serialization of bbat to AllocationTable::Big.



180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
 
# File 'lib/ole/storage/base.rb', line 180

def flush
  # update root dirent, and flatten dirent tree
  @root.name = 'Root Entry'
  @root.first_block = @sb_file.first_block
  @root.size = @sb_file.size
  @dirents = @root.flatten

  # serialize the dirents using the bbat
  RangesIOResizeable.open @bbat, 'w', :first_block => @header.dirent_start do |io|
    io.write @dirents.map { |dirent| dirent.to_s }.join
    padding = (io.size / @bbat.block_size.to_f).ceil * @bbat.block_size - io.size
    io.write 0.chr * padding
    @header.dirent_start = io.first_block
  end

  # serialize the sbat
  # perhaps the blocks used by the sbat should be marked with BAT?
  RangesIOResizeable.open @bbat, 'w', :first_block => @header.sbat_start do |io|
    io.write @sbat.to_s
    @header.sbat_start = io.first_block
    @header.num_sbat = @bbat.chain(@header.sbat_start).length
  end

  # create RangesIOResizeable hooked up to the bbat. use that to claim bbat blocks using
  # truncate. then when its time to write, convert that chain and some chunk of blocks at
  # the end, into META_BAT blocks. write out the chain, and those meta bat blocks, and its
  # done.
  # this is perhaps not good, as we reclaim all bat blocks here, which
  # may include the sbat we just wrote. FIXME
  @bbat.map! do |b|
    b == AllocationTable::BAT || b == AllocationTable::META_BAT ? AllocationTable::AVAIL : b
  end

  # currently we use a loop. this could be better, but basically,
  # the act of writing out the bat, itself requires blocks which get
  # recorded in the bat.
  #
  # i'm sure that there'd be some simpler closed form solution to this. solve
  # recursive func:
  #
  #   num_mbat_blocks = ceil(max((mbat_len - 109) * 4 / block_size, 0))
  #   bbat_len = initial_bbat_len + num_mbat_blocks
  #   mbat_len = ceil(bbat_len * 4 / block_size)
  #
  # the actual bbat allocation table is itself stored throughout the file, and that chain
  # is stored in the initial blocks, and the mbat blocks.
  num_mbat_blocks = 0
  io = RangesIOResizeable.new @bbat, 'w', :first_block => AllocationTable::EOC
  # truncate now, so that we can simplify size calcs - the mbat blocks will be appended in a
  # contiguous chunk at the end.
  # hmmm, i think this truncate should be matched with a truncate of the underlying io. if you
  # delete a lot of stuff, and free up trailing blocks, the file size never shrinks. this can
  # be fixed easily, add an io truncate
  @bbat.truncate!
  @io.truncate @bbat.block_size * (@bbat.length + 1)
  while true
    # get total bbat size. equivalent to @bbat.to_s.length, but for the factoring in of
    # the mbat blocks. we can't just add the mbat blocks directly to the bbat, as as this iteration
    # progresses, more blocks may be needed for the bat itself (if there are no more gaps), and the
    # mbat must remain contiguous.
    bbat_data_len = ((@bbat.length + num_mbat_blocks) * 4 / @bbat.block_size.to_f).ceil * @bbat.block_size
    # now storing the excess mbat blocks also increases the size of the bbat:
    new_num_mbat_blocks = ([bbat_data_len / @bbat.block_size - 109, 0].max * 4 / (@bbat.block_size.to_f - 4)).ceil
    if new_num_mbat_blocks != num_mbat_blocks
      # need more space for the mbat.
      num_mbat_blocks = new_num_mbat_blocks
    elsif io.size != bbat_data_len
      # need more space for the bat
      # this may grow the bbat, depending on existing available blocks
      io.truncate bbat_data_len
    else
      break
    end
  end

  # now extract the info we want:
  ranges = io.ranges
  bbat_chain = @bbat.chain io.first_block
  io.close
  bbat_chain.each { |b| @bbat[b] = AllocationTable::BAT }
  # tack on the mbat stuff
  @header.num_bat = bbat_chain.length
  mbat_blocks = (0...num_mbat_blocks).map do
    block = @bbat.free_block
    @bbat[block] = AllocationTable::META_BAT
    block
  end
  @header.mbat_start = mbat_blocks.first || AllocationTable::EOC

  # now finally write the bbat, using a not resizable io.
  # the mode here will be 'r', which allows write atm. 
  RangesIO.open(@io, :ranges => ranges) { |f| f.write @bbat.to_s }

  # this is the mbat. pad it out.
  bbat_chain += [AllocationTable::AVAIL] * [109 - bbat_chain.length, 0].max
  @header.num_mbat = num_mbat_blocks
  if num_mbat_blocks != 0
    # write out the mbat blocks now. first of all, where are they going to be?
    mbat_data = bbat_chain[109..-1]
    # expand the mbat_data to include the linked list forward pointers.
    mbat_data = mbat_data.to_enum(:each_slice, @bbat.block_size / 4 - 1).to_a.
      zip(mbat_blocks[1..-1] + [nil]).map { |a, b| b ? a + [b] : a }
    # pad out the last one.
    mbat_data.last.push(*([AllocationTable::AVAIL] * (@bbat.block_size / 4 - mbat_data.last.length)))
    RangesIO.open @io, :ranges => @bbat.ranges(mbat_blocks) do |f|
      f.write mbat_data.flatten.pack('V*')
    end
  end

  # now seek back and write the header out
  @io.seek 0
  @io.write @header.to_s + bbat_chain[0, 109].pack('V*')
  @io.flush
end

#inspect ⇒ Object 



338
339
340
 
# File 'lib/ole/storage/base.rb', line 338

def inspect
  "#<#{self.class} io=#{@io.inspect} root=#{@root.inspect}>"
end

#load ⇒ Object

load document from file.

TODO: implement various allocationtable checks, maybe as a AllocationTable#fsck function :)

  1. reterminate any chain not ending in EOC. compare file size with actually allocated blocks per file.

  2. pass through all chain heads looking for collisions, and making sure nothing points to them (ie they are really heads). in both sbat and mbat

  3. we know the locations of the bbat data, and mbat data. ensure that there are placeholder blocks in the bat for them.

  4. maybe a check of excess data. if there is data outside the bbat.truncate.length + 1 * block_size, (eg what is used for truncate in #flush), then maybe add some sort of message about that. it will be automatically thrown away at close time.



107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
 
# File 'lib/ole/storage/base.rb', line 107

def load
  # we always read 512 for the header block. if the block size ends up being different,
  # what happens to the 109 fat entries. are there more/less entries?
  @io.rewind
  header_block = @io.read 512
  @header = Header.new header_block

  # create an empty bbat.
  @bbat = AllocationTable::Big.new self
  bbat_chain = header_block[Header::SIZE..-1].unpack 'V*'
  mbat_block = @header.mbat_start
  @header.num_mbat.times do
    blocks = @bbat.read([mbat_block]).unpack 'V*'
    mbat_block = blocks.pop
    bbat_chain += blocks
  end
  # am i using num_bat in the right way?
  @bbat.load @bbat.read(bbat_chain[0, @header.num_bat])
  
  # get block chain for directories, read it, then split it into chunks and load the
  # directory entries. semantics changed - used to cut at first dir where dir.type == 0
  @dirents = @bbat.read(@header.dirent_start).to_enum(:each_chunk, Dirent::SIZE).
    map { |str| Dirent.new self, str }

  # now reorder from flat into a tree
  # links are stored in some kind of balanced binary tree
  # check that everything is visited at least, and at most once
  # similarly with the blocks of the file.
  # was thinking of moving this to Dirent.to_tree instead.
  class << @dirents
    def to_tree idx=0
      return [] if idx == Dirent::EOT
      d = self[idx]
      to_tree(d.child).each { |child| d << child }
      raise FormatError, "directory #{d.inspect} used twice" if d.idx
      d.idx = idx
      to_tree(d.prev) + [d] + to_tree(d.next)
    end
  end

  @root = @dirents.to_tree.first
  @dirents.reject! { |d| d.type_id == 0 }
  # silence this warning by default, its not really important (issue #5).
  # fairly common one appears to be "R" (from office OS X?) which smells
  # like some kind of UTF16 snafu, but scottwillson also has had some kanji...
  #Log.warn "root name was #{@root.name.inspect}" unless @root.name == 'Root Entry'
  unused = @dirents.reject(&:idx).length
  Log.warn "#{unused} unused directories" if unused > 0

  # FIXME i don't currently use @header.num_sbat which i should
  # hmm. nor do i write it. it means what exactly again?
  # which mode to use here?
  @sb_file = RangesIOResizeable.new @bbat, :first_block => @root.first_block, :size => @root.size
  @sbat = AllocationTable::Small.new self
  @sbat.load @bbat.read(@header.sbat_start)
end

#meta_data ⇒ Object 



145
146
147
 
# File 'lib/ole/storage/meta_data.rb', line 145

def meta_data
  @meta_data ||= MetaData.new(self)
end

#repack(temp = :file) ⇒ Object

could be useful with mis-behaving ole documents. or to just clean them up.



311
312
313
314
315
316
317
318
319
320
321
 
# File 'lib/ole/storage/base.rb', line 311

def repack temp=:file
  case temp
  when :file
    Tempfile.open 'ole-repack' do |io|
      io.binmode
      repack_using_io io
    end
  when :mem;  StringIO.open(''.dup, &method(:repack_using_io))
  else raise ArgumentError, "unknown temp backing #{temp.inspect}"
  end
end

#repack_using_io(temp_io) ⇒ Object 



323
324
325
326
327
328
329
330
331
 
# File 'lib/ole/storage/base.rb', line 323

def repack_using_io temp_io
  @io.rewind
  IO.copy @io, temp_io
  clear
  Storage.open temp_io, nil, @params do |temp_ole|
    #temp_ole.root.type = :dir
    Dirent.copy temp_ole.root, root
  end
end