Class: SqlPartitioner::PartitionsManager

Inherits:
BasePartitionsManager show all
Defined in:
lib/sql_partitioner/partitions_manager.rb

Overview

Performs partitioning operations against a table.

Constant Summary collapse

VALID_PARTITION_SIZE_UNITS = 
[:months, :days]

Constants inherited from BasePartitionsManager

BasePartitionsManager::FUTURE_PARTITION_NAME, BasePartitionsManager::FUTURE_PARTITION_VALUE

Instance Attribute Summary

Attributes inherited from BasePartitionsManager

#adapter, #current_timestamp, #logger, #table_name

Instance Method Summary collapse

Methods inherited from BasePartitionsManager

#drop_partitions, #initialize, #initialize_partitioning, #log, #name_from_timestamp, #reorg_future_partition

Constructor Details

This class inherits a constructor from SqlPartitioner::BasePartitionsManager

Instance Method Details

#append_partition_intervals(partition_size_unit, partition_size, days_into_future = 30, dry_run = false) ⇒ Hash

Wrapper around append partition to add a partition to end with the given window size

Parameters:

  • partition_size_unit: (Symbol) —
    :days, :months
  • partition_size, (Fixnum) —

    intervals covered by the new partition

  • days_into_future, (Fixnum) —

    how many days into the future need to be covered by partitions

  • dry_run, (Boolean) —

    Defaults to false. If true, query wont be executed.

Returns:

  • (Hash) —

    partition_data hash of the partitions appended

Raises:

  • (ArgumentError) —

    if window size is nil or not greater than 0



77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
 
# File 'lib/sql_partitioner/partitions_manager.rb', line 77

def append_partition_intervals(partition_size_unit, partition_size, days_into_future = 30, dry_run = false)
  partitions = Partition.all(adapter, table_name)
  if partitions.blank? || partitions.non_future_partitions.blank?
    raise "partitions must be properly initialized before appending"
  end
  latest_partition = partitions.latest_partition

  new_partition_data = partitions_to_append(latest_partition.timestamp, partition_size_unit, partition_size, days_into_future)

  if new_partition_data.empty?
    msg = "Append: No-Op - Latest Partition Time of #{latest_partition.timestamp}, " +
          "i.e. #{Time.at(@tuc.to_seconds(latest_partition.timestamp))} covers >= #{days_into_future} days_into_future"
  else
    msg = "Append: Appending the following new partitions: #{new_partition_data.inspect}"
    reorg_future_partition(new_partition_data, dry_run)
  end

  log(msg)

  new_partition_data
end

#drop_partitions_older_than(timestamp, dry_run = false) ⇒ Array

drop partitions that are older than the given timestamp

Parameters:

  • timestamp (Fixnum) —

    partitions older than this timestamp will be dropped

  • dry_run, (Boolean) —

    Defaults to false. If true, query wont be executed.

Returns:

  • (Array) —

    an array of partition names that were dropped



112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/sql_partitioner/partitions_manager.rb', line 112

def drop_partitions_older_than(timestamp, dry_run = false)
  partitions = Partition.all(adapter, table_name).older_than_timestamp(timestamp)
  partition_names = partitions.map(&:name)

  if partition_names.empty?
    msg = "Drop: No-Op - No partitions older than #{timestamp}, i.e. #{Time.at(@tuc.to_seconds(timestamp))} to drop"
  else
    msg = "Drop: Dropped partitions: #{partition_names.inspect}"
    drop_partitions(partition_names, dry_run)
  end

  log(msg)

  partition_names
end

#drop_partitions_older_than_in_days(days_from_now, dry_run = false) ⇒ Object

drop partitions that are older than days(input) from now

Parameters:

  • days_from_now (Fixnum)
  • dry_run, (Boolean) —

    Defaults to false. If true, query wont be executed.



102
103
104
105
 
# File 'lib/sql_partitioner/partitions_manager.rb', line 102

def drop_partitions_older_than_in_days(days_from_now, dry_run = false)
  timestamp = self.current_timestamp - @tuc.from_days(days_from_now)
  drop_partitions_older_than(timestamp, dry_run)
end

#initialize_partitioning_in_intervals(days_into_future, partition_size_unit = :months, partition_size = 1, dry_run = false) ⇒ Boolean, String

Initialize the partitions based on intervals relative to current timestamp. Partition size specified by (partition_size, partition_size_unit), i.e. 1 month. Partitions will be created as needed to cover days_into_future.

Parameters:

  • days_into_future (Fixnum) —

    Number of days into the future from current_timestamp

  • partition_size_unit (Symbol) (defaults to: :months) —

    one of: [:months, :days]

  • partition_size (Fixnum) (defaults to: 1) —

    size of partition (in terms of partition_size_unit), i.e. 3 month

  • dry_run (Boolean) (defaults to: false) —

    Defaults to false. If true, query wont be executed.

Returns:

  • (Boolean) —

    true if not dry run

  • (String) —

    sql to initialize partitions if dry run is true

Raises:

  • (ArgumentError) —

    if days is not array or if one of the days is not integer



18
19
20
21
 
# File 'lib/sql_partitioner/partitions_manager.rb', line 18

def initialize_partitioning_in_intervals(days_into_future, partition_size_unit = :months, partition_size = 1, dry_run = false)
  partition_data = partitions_to_append(@current_timestamp, partition_size_unit, partition_size, days_into_future)
  initialize_partitioning(partition_data, dry_run)
end

#partitions_to_append(partition_start_timestamp, partition_size_unit, partition_size, days_into_future) ⇒ Hash

Get partition to add a partition to end with the given window size

Parameters:

  • partition_start_timestamp (Fixnum)
  • partition_size_unit: (Symbol) —
    :days, :months
  • partition_size, (Fixnum) —

    size of partition (in terms of partition_size_unit)

  • partitions_into_future, (Fixnum) —

    how many partitions into the future should be covered

Returns:

  • (Hash) —

    partition_data hash



31
32
33
34
35
36
 
# File 'lib/sql_partitioner/partitions_manager.rb', line 31

def partitions_to_append(partition_start_timestamp, partition_size_unit, partition_size, days_into_future)
  _validate_positive_fixnum(:days_into_future, days_into_future)

  end_timestamp = @tuc.advance(current_timestamp, :days, days_into_future)
  partitions_to_append_by_ts_range(partition_start_timestamp, end_timestamp, partition_size_unit, partition_size)
end

#partitions_to_append_by_ts_range(partition_start_timestamp, end_timestamp, partition_size_unit, partition_size) ⇒ Hash

Get partition_data hash based on the last partition’s timestamp and covering end_timestamp

Parameters:

  • partition_start_timestamp, (Fixnum) —

    timestamp of last partition

  • end_timestamp, (Fixnum) —

    timestamp which the newest partition needs to include

  • partition_size_unit: (Symbol) —
    :days, :months
  • partition_size, (Fixnum) —

    intervals covered by the new partition

Returns:

  • (Hash) —

    partition_data hash



46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/sql_partitioner/partitions_manager.rb', line 46

def partitions_to_append_by_ts_range(partition_start_timestamp, end_timestamp, partition_size_unit, partition_size)
  if partition_size_unit.nil? || !VALID_PARTITION_SIZE_UNITS.include?(partition_size_unit)
    _raise_arg_err "partition_size_unit must be one of: #{VALID_PARTITION_SIZE_UNITS.inspect}"
  end

  _validate_positive_fixnum(:partition_size, partition_size)
  _validate_positive_fixnum(:partition_start_timestamp, partition_start_timestamp)
  _validate_positive_fixnum(:end_timestamp, end_timestamp)

  timestamp = partition_start_timestamp

  partitions_to_append = {}
  while timestamp < end_timestamp
    timestamp = @tuc.advance(timestamp, partition_size_unit, partition_size)

    partition_name = name_from_timestamp(timestamp)
    partitions_to_append[partition_name] = timestamp
  end

  partitions_to_append
end