Class: PdfTableExtractor
- Inherits:
-
Object
- Object
- PdfTableExtractor
- Defined in:
- lib/pdf_table_extractor/pdf_table_extractor.rb,
lib/pdf_table_extractor/version.rb
Overview
PdfTableExtractor extracts tables from PDF text using spacing and position heuristics.
Constant Summary collapse
- VERSION =
'0.1.0'.freeze
Instance Attribute Summary collapse
-
#merged_rows ⇒ Array<PdfTableExtractorRow>
readonly
Rows merged into tables.
-
#options ⇒ Hash
readonly
Configuration options.
-
#rows ⇒ Array<PdfTableExtractorRow>
readonly
Raw rows parsed from text.
Instance Method Summary collapse
-
#extract_tables ⇒ void
Extracts tables from the PDF and stores them in @merged_rows.
-
#initialize(pdf_path = nil, reader: nil, options: {}) ⇒ PdfTableExtractor
constructor
A new instance of PdfTableExtractor.
-
#result ⇒ Array<Array<Hash>>
Returns the extracted cells as arrays per merged row.
Constructor Details
#initialize(pdf_path = nil, reader: nil, options: {}) ⇒ PdfTableExtractor
Returns a new instance of PdfTableExtractor.
25 26 27 28 29 30 31 32 33 34 35 |
# File 'lib/pdf_table_extractor/pdf_table_extractor.rb', line 25 def initialize(pdf_path = nil, reader: nil, options: {}) @reader = reader || PDF::Reader.new(pdf_path) @options = @options[:remove_page_headers] = true unless @options.key?(:remove_page_headers) @options[:remove_page_footers] = true unless @options.key?(:remove_page_footers) @options[:remove_pagination_from_header] = false unless @options.key?(:remove_pagination_from_header) @options[:remove_pagination_from_footer] = false unless @options.key?(:remove_pagination_from_footer) @options[:remove_empty_lines] = true unless @options.key?(:remove_empty_lines) @options[:position_tolerance] = 2 unless @options.key?(:position_tolerance) @merged_rows = [] end |
Instance Attribute Details
#merged_rows ⇒ Array<PdfTableExtractorRow> (readonly)
Returns rows merged into tables.
13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 |
# File 'lib/pdf_table_extractor/pdf_table_extractor.rb', line 13 class PdfTableExtractor attr_reader :rows, :merged_rows, :options # @param pdf_path [String, nil] Path to the PDF file (optional when reader is provided) # @param reader [PDF::Reader, nil] Pre-initialized PDF::Reader instance # @param options [Hash] Configuration options # @option options [Boolean] :remove_page_headers (true) Remove common leading lines across pages # @option options [Boolean] :remove_page_footers (true) Remove common trailing lines across pages # @option options [Boolean, Integer] :remove_pagination_from_header (false) true or line number from top # @option options [Boolean, Integer] :remove_pagination_from_footer (false) true or line number from bottom # @option options [Boolean] :remove_empty_lines (true) Remove empty lines from the extracted text # @option options [Integer] :position_tolerance (2) Tolerance for matching column positions def initialize(pdf_path = nil, reader: nil, options: {}) @reader = reader || PDF::Reader.new(pdf_path) @options = @options[:remove_page_headers] = true unless @options.key?(:remove_page_headers) @options[:remove_page_footers] = true unless @options.key?(:remove_page_footers) @options[:remove_pagination_from_header] = false unless @options.key?(:remove_pagination_from_header) @options[:remove_pagination_from_footer] = false unless @options.key?(:remove_pagination_from_footer) @options[:remove_empty_lines] = true unless @options.key?(:remove_empty_lines) @options[:position_tolerance] = 2 unless @options.key?(:position_tolerance) @merged_rows = [] end # Extracts tables from the PDF and stores them in @merged_rows # @return [void] def extract_tables pages = all_pages pages = remove_pagination(pages) if @options[:remove_pagination_from_header] || @options[:remove_pagination_from_footer] pages = remove_common_leading_lines(pages) if @options[:remove_page_headers] pages = remove_common_trailing_lines(pages) if @options[:remove_page_footers] lines = pages&.flatten lines = lines&.reject { |l| l.strip.empty? } if @options[:remove_empty_lines] @rows = lines&.map&.with_index do |line, index| cells, positions = parse_line_to_cells(line) PdfTableExtractorRow.new(self, cells, positions, index) end process_rows end # Returns the extracted cells as arrays per merged row. # @return [Array<Array<Hash>>] Array of rows, each row is an array of cells with :text and :position def result @merged_rows.map(&:cells) end private # @return [Array<Array<String>>] Array of pages, each page is an array of lines def all_pages all_pages_texts.map { |page_text| page_text.lines.map(&:chomp) } end # @return [Array<String>] Array of page texts def all_pages_texts @reader.pages.map { |page| page.text } end # Remove pagination lines from headers and footers. # @param pages [Array<Array<String>>] # @return [Array<Array<String>>] def remove_pagination(pages) return pages if pages.empty? || pages.length == 1 if @options[:remove_pagination_from_header] if @options[:remove_pagination_from_header].is_a?(Integer) index = @options[:remove_pagination_from_header] - 1 pages.each do |lines| if lines.length > index && is_pagination?(lines[index]) lines.delete_at(index) end end else [1..5].each do |index| if pages.all? { |lines| lines.length > index && is_pagination?(lines[index - 1]) } pages.each { |lines| lines.delete_at(index - 1) } break end end end end if @options[:remove_pagination_from_footer] if @options[:remove_pagination_from_footer].is_a?(Integer) index = @options[:remove_pagination_from_footer] pages.each do |lines| if lines.length > index && is_pagination?(lines[-index]) lines.delete_at(-index) end end else [1..5].each do |index| if pages.all? { |lines| lines.length > index && is_pagination?(lines[-index]) } pages.each { |lines| lines.delete_at(-index) } break end end end end pages end # Remove common leading lines across pages. # @param pages [Array<Array<String>>] # @return [Array<Array<String>>] def remove_common_leading_lines(pages) return pages if pages.empty? || pages.length == 1 pages.each(&:shift) while same_leading_line?(pages) pages end # Remove common trailing lines across pages. # @param pages [Array<Array<String>>] # @return [Array<Array<String>>] def remove_common_trailing_lines(pages) return pages if pages.empty? || pages.length == 1 pages.each(&:pop) while same_trailing_line?(pages) pages end # Parse a line into cells using runs of multiple spaces as separators. # @param line [String] # @return [Array<(Array<Hash>, Array<Integer>)>] cells and positions def parse_line_to_cells(line) if has_consecutive_spaces?(line) cells = [] position = 0 positions = [] line.split(/(\s{2,})/).each do |text| if has_consecutive_spaces?(text) position += text.length elsif !text.empty? cells << {text:, position:} positions << position position += text.length end end [cells, positions] else [[{text: line.strip, position: 0}], [0]] end end # Process parsed rows into merged table rows. # @return [void] def process_rows @merged_rows = [] @rows.each do |row| row.transform_to_single_cell! if row.incongruent_with_neighbours? if @merged_rows.empty? || !row.congruent_with_last_merged? @merged_rows << PdfTableExtractorRow.new(self, row.cells, row.positions, nil, true) else @merged_rows.last.merge!(row) end end end # @param pages [Array<Array<String>>] # @return [Boolean] def same_leading_line?(pages) pages.all? { |lines| lines.any? } && pages.all? { |lines| lines[0] == pages[0][0] } end # @param pages [Array<Array<String>>] # @return [Boolean] def same_trailing_line?(pages) pages.all? { |lines| lines.any? } && pages.all? { |lines| lines[-1] == pages[0][-1] } end # @param text [String] # @return [Boolean] def has_consecutive_spaces?(text) text.match?(/\s{2,}/) end # @param line [String, nil] # @return [Boolean] def is_pagination?(line) line&.strip&.match?(/^.*\d+$/) end end |
#options ⇒ Hash (readonly)
Returns configuration options.
13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 |
# File 'lib/pdf_table_extractor/pdf_table_extractor.rb', line 13 class PdfTableExtractor attr_reader :rows, :merged_rows, :options # @param pdf_path [String, nil] Path to the PDF file (optional when reader is provided) # @param reader [PDF::Reader, nil] Pre-initialized PDF::Reader instance # @param options [Hash] Configuration options # @option options [Boolean] :remove_page_headers (true) Remove common leading lines across pages # @option options [Boolean] :remove_page_footers (true) Remove common trailing lines across pages # @option options [Boolean, Integer] :remove_pagination_from_header (false) true or line number from top # @option options [Boolean, Integer] :remove_pagination_from_footer (false) true or line number from bottom # @option options [Boolean] :remove_empty_lines (true) Remove empty lines from the extracted text # @option options [Integer] :position_tolerance (2) Tolerance for matching column positions def initialize(pdf_path = nil, reader: nil, options: {}) @reader = reader || PDF::Reader.new(pdf_path) @options = @options[:remove_page_headers] = true unless @options.key?(:remove_page_headers) @options[:remove_page_footers] = true unless @options.key?(:remove_page_footers) @options[:remove_pagination_from_header] = false unless @options.key?(:remove_pagination_from_header) @options[:remove_pagination_from_footer] = false unless @options.key?(:remove_pagination_from_footer) @options[:remove_empty_lines] = true unless @options.key?(:remove_empty_lines) @options[:position_tolerance] = 2 unless @options.key?(:position_tolerance) @merged_rows = [] end # Extracts tables from the PDF and stores them in @merged_rows # @return [void] def extract_tables pages = all_pages pages = remove_pagination(pages) if @options[:remove_pagination_from_header] || @options[:remove_pagination_from_footer] pages = remove_common_leading_lines(pages) if @options[:remove_page_headers] pages = remove_common_trailing_lines(pages) if @options[:remove_page_footers] lines = pages&.flatten lines = lines&.reject { |l| l.strip.empty? } if @options[:remove_empty_lines] @rows = lines&.map&.with_index do |line, index| cells, positions = parse_line_to_cells(line) PdfTableExtractorRow.new(self, cells, positions, index) end process_rows end # Returns the extracted cells as arrays per merged row. # @return [Array<Array<Hash>>] Array of rows, each row is an array of cells with :text and :position def result @merged_rows.map(&:cells) end private # @return [Array<Array<String>>] Array of pages, each page is an array of lines def all_pages all_pages_texts.map { |page_text| page_text.lines.map(&:chomp) } end # @return [Array<String>] Array of page texts def all_pages_texts @reader.pages.map { |page| page.text } end # Remove pagination lines from headers and footers. # @param pages [Array<Array<String>>] # @return [Array<Array<String>>] def remove_pagination(pages) return pages if pages.empty? || pages.length == 1 if @options[:remove_pagination_from_header] if @options[:remove_pagination_from_header].is_a?(Integer) index = @options[:remove_pagination_from_header] - 1 pages.each do |lines| if lines.length > index && is_pagination?(lines[index]) lines.delete_at(index) end end else [1..5].each do |index| if pages.all? { |lines| lines.length > index && is_pagination?(lines[index - 1]) } pages.each { |lines| lines.delete_at(index - 1) } break end end end end if @options[:remove_pagination_from_footer] if @options[:remove_pagination_from_footer].is_a?(Integer) index = @options[:remove_pagination_from_footer] pages.each do |lines| if lines.length > index && is_pagination?(lines[-index]) lines.delete_at(-index) end end else [1..5].each do |index| if pages.all? { |lines| lines.length > index && is_pagination?(lines[-index]) } pages.each { |lines| lines.delete_at(-index) } break end end end end pages end # Remove common leading lines across pages. # @param pages [Array<Array<String>>] # @return [Array<Array<String>>] def remove_common_leading_lines(pages) return pages if pages.empty? || pages.length == 1 pages.each(&:shift) while same_leading_line?(pages) pages end # Remove common trailing lines across pages. # @param pages [Array<Array<String>>] # @return [Array<Array<String>>] def remove_common_trailing_lines(pages) return pages if pages.empty? || pages.length == 1 pages.each(&:pop) while same_trailing_line?(pages) pages end # Parse a line into cells using runs of multiple spaces as separators. # @param line [String] # @return [Array<(Array<Hash>, Array<Integer>)>] cells and positions def parse_line_to_cells(line) if has_consecutive_spaces?(line) cells = [] position = 0 positions = [] line.split(/(\s{2,})/).each do |text| if has_consecutive_spaces?(text) position += text.length elsif !text.empty? cells << {text:, position:} positions << position position += text.length end end [cells, positions] else [[{text: line.strip, position: 0}], [0]] end end # Process parsed rows into merged table rows. # @return [void] def process_rows @merged_rows = [] @rows.each do |row| row.transform_to_single_cell! if row.incongruent_with_neighbours? if @merged_rows.empty? || !row.congruent_with_last_merged? @merged_rows << PdfTableExtractorRow.new(self, row.cells, row.positions, nil, true) else @merged_rows.last.merge!(row) end end end # @param pages [Array<Array<String>>] # @return [Boolean] def same_leading_line?(pages) pages.all? { |lines| lines.any? } && pages.all? { |lines| lines[0] == pages[0][0] } end # @param pages [Array<Array<String>>] # @return [Boolean] def same_trailing_line?(pages) pages.all? { |lines| lines.any? } && pages.all? { |lines| lines[-1] == pages[0][-1] } end # @param text [String] # @return [Boolean] def has_consecutive_spaces?(text) text.match?(/\s{2,}/) end # @param line [String, nil] # @return [Boolean] def is_pagination?(line) line&.strip&.match?(/^.*\d+$/) end end |
#rows ⇒ Array<PdfTableExtractorRow> (readonly)
Returns raw rows parsed from text.
13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 |
# File 'lib/pdf_table_extractor/pdf_table_extractor.rb', line 13 class PdfTableExtractor attr_reader :rows, :merged_rows, :options # @param pdf_path [String, nil] Path to the PDF file (optional when reader is provided) # @param reader [PDF::Reader, nil] Pre-initialized PDF::Reader instance # @param options [Hash] Configuration options # @option options [Boolean] :remove_page_headers (true) Remove common leading lines across pages # @option options [Boolean] :remove_page_footers (true) Remove common trailing lines across pages # @option options [Boolean, Integer] :remove_pagination_from_header (false) true or line number from top # @option options [Boolean, Integer] :remove_pagination_from_footer (false) true or line number from bottom # @option options [Boolean] :remove_empty_lines (true) Remove empty lines from the extracted text # @option options [Integer] :position_tolerance (2) Tolerance for matching column positions def initialize(pdf_path = nil, reader: nil, options: {}) @reader = reader || PDF::Reader.new(pdf_path) @options = @options[:remove_page_headers] = true unless @options.key?(:remove_page_headers) @options[:remove_page_footers] = true unless @options.key?(:remove_page_footers) @options[:remove_pagination_from_header] = false unless @options.key?(:remove_pagination_from_header) @options[:remove_pagination_from_footer] = false unless @options.key?(:remove_pagination_from_footer) @options[:remove_empty_lines] = true unless @options.key?(:remove_empty_lines) @options[:position_tolerance] = 2 unless @options.key?(:position_tolerance) @merged_rows = [] end # Extracts tables from the PDF and stores them in @merged_rows # @return [void] def extract_tables pages = all_pages pages = remove_pagination(pages) if @options[:remove_pagination_from_header] || @options[:remove_pagination_from_footer] pages = remove_common_leading_lines(pages) if @options[:remove_page_headers] pages = remove_common_trailing_lines(pages) if @options[:remove_page_footers] lines = pages&.flatten lines = lines&.reject { |l| l.strip.empty? } if @options[:remove_empty_lines] @rows = lines&.map&.with_index do |line, index| cells, positions = parse_line_to_cells(line) PdfTableExtractorRow.new(self, cells, positions, index) end process_rows end # Returns the extracted cells as arrays per merged row. # @return [Array<Array<Hash>>] Array of rows, each row is an array of cells with :text and :position def result @merged_rows.map(&:cells) end private # @return [Array<Array<String>>] Array of pages, each page is an array of lines def all_pages all_pages_texts.map { |page_text| page_text.lines.map(&:chomp) } end # @return [Array<String>] Array of page texts def all_pages_texts @reader.pages.map { |page| page.text } end # Remove pagination lines from headers and footers. # @param pages [Array<Array<String>>] # @return [Array<Array<String>>] def remove_pagination(pages) return pages if pages.empty? || pages.length == 1 if @options[:remove_pagination_from_header] if @options[:remove_pagination_from_header].is_a?(Integer) index = @options[:remove_pagination_from_header] - 1 pages.each do |lines| if lines.length > index && is_pagination?(lines[index]) lines.delete_at(index) end end else [1..5].each do |index| if pages.all? { |lines| lines.length > index && is_pagination?(lines[index - 1]) } pages.each { |lines| lines.delete_at(index - 1) } break end end end end if @options[:remove_pagination_from_footer] if @options[:remove_pagination_from_footer].is_a?(Integer) index = @options[:remove_pagination_from_footer] pages.each do |lines| if lines.length > index && is_pagination?(lines[-index]) lines.delete_at(-index) end end else [1..5].each do |index| if pages.all? { |lines| lines.length > index && is_pagination?(lines[-index]) } pages.each { |lines| lines.delete_at(-index) } break end end end end pages end # Remove common leading lines across pages. # @param pages [Array<Array<String>>] # @return [Array<Array<String>>] def remove_common_leading_lines(pages) return pages if pages.empty? || pages.length == 1 pages.each(&:shift) while same_leading_line?(pages) pages end # Remove common trailing lines across pages. # @param pages [Array<Array<String>>] # @return [Array<Array<String>>] def remove_common_trailing_lines(pages) return pages if pages.empty? || pages.length == 1 pages.each(&:pop) while same_trailing_line?(pages) pages end # Parse a line into cells using runs of multiple spaces as separators. # @param line [String] # @return [Array<(Array<Hash>, Array<Integer>)>] cells and positions def parse_line_to_cells(line) if has_consecutive_spaces?(line) cells = [] position = 0 positions = [] line.split(/(\s{2,})/).each do |text| if has_consecutive_spaces?(text) position += text.length elsif !text.empty? cells << {text:, position:} positions << position position += text.length end end [cells, positions] else [[{text: line.strip, position: 0}], [0]] end end # Process parsed rows into merged table rows. # @return [void] def process_rows @merged_rows = [] @rows.each do |row| row.transform_to_single_cell! if row.incongruent_with_neighbours? if @merged_rows.empty? || !row.congruent_with_last_merged? @merged_rows << PdfTableExtractorRow.new(self, row.cells, row.positions, nil, true) else @merged_rows.last.merge!(row) end end end # @param pages [Array<Array<String>>] # @return [Boolean] def same_leading_line?(pages) pages.all? { |lines| lines.any? } && pages.all? { |lines| lines[0] == pages[0][0] } end # @param pages [Array<Array<String>>] # @return [Boolean] def same_trailing_line?(pages) pages.all? { |lines| lines.any? } && pages.all? { |lines| lines[-1] == pages[0][-1] } end # @param text [String] # @return [Boolean] def has_consecutive_spaces?(text) text.match?(/\s{2,}/) end # @param line [String, nil] # @return [Boolean] def is_pagination?(line) line&.strip&.match?(/^.*\d+$/) end end |
Instance Method Details
#extract_tables ⇒ void
This method returns an undefined value.
Extracts tables from the PDF and stores them in @merged_rows
39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 |
# File 'lib/pdf_table_extractor/pdf_table_extractor.rb', line 39 def extract_tables pages = all_pages pages = remove_pagination(pages) if @options[:remove_pagination_from_header] || @options[:remove_pagination_from_footer] pages = remove_common_leading_lines(pages) if @options[:remove_page_headers] pages = remove_common_trailing_lines(pages) if @options[:remove_page_footers] lines = pages&.flatten lines = lines&.reject { |l| l.strip.empty? } if @options[:remove_empty_lines] @rows = lines&.map&.with_index do |line, index| cells, positions = parse_line_to_cells(line) PdfTableExtractorRow.new(self, cells, positions, index) end process_rows end |
#result ⇒ Array<Array<Hash>>
Returns the extracted cells as arrays per merged row.
59 60 61 |
# File 'lib/pdf_table_extractor/pdf_table_extractor.rb', line 59 def result @merged_rows.map(&:cells) end |