Class: ActiveRoad::OsmPbfImporterLevelDb

Inherits:
Object
  • Object
show all
Includes:
OsmPbfImporter
Defined in:
app/models/active_road/osm_pbf_importer_level_db.rb

Defined Under Namespace

Classes: SimpleWay

Constant Summary collapse

@@csv_batch_size = 
100000

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from OsmPbfImporter

#bike?, #car?, #extract_relation_polygon, included, #join_way, #join_ways, #pedestrian?, #physical_road_conditionnal_costs, #required_relation?, #required_way?, #selected_tags, #train?

Constructor Details

#initialize(pbf_file, split_ways = false, nodes_database_path = "/tmp/osm_pbf_nodes_leveldb", ways_database_path = "/tmp/osm_pbf_ways_leveldb") ⇒ OsmPbfImporterLevelDb

Returns a new instance of OsmPbfImporterLevelDb.



13
14
15
16
17
18
19
20
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 13

def initialize(pbf_file, split_ways = false, nodes_database_path = "/tmp/osm_pbf_nodes_leveldb", ways_database_path = "/tmp/osm_pbf_ways_leveldb")
  @pbf_file = pbf_file
  @split_ways = split_ways
  @nodes_database_path = nodes_database_path
  @ways_database_path = ways_database_path
  @junctions_database_path = "/tmp/osm_pbf_junctions_leveldb"
  @physical_roads_database_path = "/tmp/osm_pbf_physical_roads_leveldb"
end

Instance Attribute Details

#junctions_database_path ⇒ Object (readonly)

Returns the value of attribute junctions_database_path.



11
12
13
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 11

def junctions_database_path
  @junctions_database_path
end

#nodes_database_path ⇒ Object (readonly)

Returns the value of attribute nodes_database_path.



11
12
13
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 11

def nodes_database_path
  @nodes_database_path
end

#pbf_file ⇒ Object (readonly)

Returns the value of attribute pbf_file.



11
12
13
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 11

def pbf_file
  @pbf_file
end

#physical_roads_database_path ⇒ Object (readonly)

Returns the value of attribute physical_roads_database_path.



11
12
13
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 11

def physical_roads_database_path
  @physical_roads_database_path
end

#split_ways ⇒ Object (readonly)

Returns the value of attribute split_ways.



11
12
13
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 11

def split_ways
  @split_ways
end

#ways_database_path ⇒ Object (readonly)

Returns the value of attribute ways_database_path.



11
12
13
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 11

def ways_database_path
  @ways_database_path
end

Instance Method Details

#backup_logical_roads_pgsql ⇒ Object 



752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 752

def backup_logical_roads_pgsql
  Rails.logger.info "Begin to backup logical roads in PostgreSql"
  start = Time.now
  logical_roads_counter = 0

  saved_name = nil
  saved_boundary = nil
  saved_logical_road = nil
  ActiveRoad::PhysicalRoad.where("physical_roads.name IS NOT NULL OR physical_roads.boundary_id IS NOT NULL").select("name,boundary_id,id").order(:boundary_id,:name).find_in_batches(batch_size: 2000) do |group|
    ActiveRoad::LogicalRoad.transaction do
      group.each do |physical_road|
        not_same_name = (saved_name != physical_road.name)
        not_same_boundary = (saved_boundary != physical_road.boundary_id)
        
        saved_name = physical_road.name if not_same_name
        saved_boundary = physical_road.boundary_id if not_same_boundary

        if not_same_name || not_same_boundary
          logical_roads_counter += 1
          saved_logical_road = ActiveRoad::LogicalRoad.create(:name => saved_name, :boundary_id => saved_boundary)
        end
        
        physical_road.update_column(:logical_road_id, saved_logical_road.id) if saved_logical_road.present?
      end
    end
  end
        
  Rails.logger.info "Finish to backup #{logical_roads_counter} logical roads in PostgreSql in #{ display_time(Time.now - start)} seconds"
end

#backup_nodes ⇒ Object 



118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 118

def backup_nodes
  Rails.logger.info "Begin to backup nodes in LevelDB nodes_database in #{nodes_database_path}"
  start = Time.now
  nodes_parser = ::PbfParser.new(pbf_file)
  nodes_counter = 0
  nodes_hash = {}

  # Process the file until it finds any node
  nodes_parser.next until nodes_parser.nodes.any?
  
  until nodes_parser.nodes.empty?
    nodes_database.batch do |batch|
      last_node = nodes_parser.nodes.last
      nodes_parser.nodes.each do |node|
        nodes_counter+= 1

        select_tags = selected_tags(node[:tags], @@nodes_selected_tags_keys)         
        batch[ node[:id].to_s ] = Marshal.dump(Node.new(node[:id].to_s, node[:lon], node[:lat], select_tags["addr:housenumber"], [], false, select_tags))      
      end
    end
    # When there's no more fileblocks to parse, #next returns false
    # This avoids an infinit loop when the last fileblock still contains ways
    break unless nodes_parser.next
  end
  Rails.logger.info "Finish to backup #{nodes_counter} nodes in LevelDB nodes_database in #{display_time(Time.now - start)} seconds"
end

#backup_relations_pgsql ⇒ Object 



672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 672

def backup_relations_pgsql
  Rails.logger.info "Begin to backup relations in PostgreSql"
  start = Time.now
  relations_parser = ::PbfParser.new(pbf_file)
  boundaries_counter = 0
  
  # traverse records by iterator
  boundary_columns = ["objectid", "geometry", "name", "admin_level", "postal_code", "insee_code"]     
  
  # Process the file until it finds any relation.
  relations_parser.next until relations_parser.relations.any?
  
  # Once it found at least one relation, iterate to find the remaining relations.
  CSV.open("/tmp/boundaries.csv", "wb:UTF-8") do |boundary_csv|       
    boundary_csv << boundary_columns

    until relations_parser.relations.empty?
      relations_parser.relations.each do |relation|
        
        if relation.key?(:tags) && required_relation?(relation[:tags])
          tags = selected_tags(relation[:tags], @@relation_selected_tags_keys)
          
          # Use tags["admin_level"] == "8" because catholic boundaries exist!!
          if tags["admin_level"] == "8" && tags["boundary"] == "administrative"
            boundaries_counter += 1
            outer_ways = {}
            inner_ways = {}
            
            begin 
              relation[:members][:ways].each do |member_way|                  
                way_data = ways_database[ member_way[:id].to_s ]
                way = nil
                nodes = []
                
                if way_data.present?
                  way = Marshal.load(way_data)
                  way.nodes.each do |node_id|
                    node = Marshal.load( nodes_database[node_id.to_s] )
                    nodes << node
                  end
                else
                  raise StandardError, "Geometry error : impossible to find way #{member_way[:id]} for relation #{tags["name"]} with id #{relation[:id]}"                      
                end
                
                if  member_way[:role] == "inner"
                  inner_ways[ member_way[:id] ] = way_geometry(nodes)
                elsif member_way[:role] == "outer"
                  outer_ways[ member_way[:id] ] = way_geometry(nodes)
                else # Fix : lot of boundaries have no tags role
                  outer_ways[ member_way[:id] ] = way_geometry(nodes)
                end
              end
              
              boundary_polygons = extract_relation_polygon(outer_ways.values, inner_ways.values)
              
              if boundary_polygons.present?
                boundary_geometry = GeoRuby::SimpleFeatures::MultiPolygon.from_polygons( boundary_polygons ).as_hex_ewkb
                
                boundary_csv << [ relation[:id], boundary_geometry, tags["name"], tags["admin_level"], tags["addr:postcode"], tags["ref:INSEE"] ]
              end
            rescue StandardError => e
              Rails.logger.error "Geometry error : impossible to build polygon for relation #{tags["name"]} with id #{relation[:id]} : #{e.message}"
            end
          end
        end
      end
      
      # When there's no more fileblocks to parse, #next returns false
      # This avoids an infinit loop when the last fileblock still contains relations
      break unless relations_parser.next                 
    end
  end
  
  ActiveRoad::Boundary.transaction do                                         
    ActiveRoad::Boundary.pg_copy_from "/tmp/boundaries.csv"        
  end
  
  Rails.logger.info  "Finish to backup #{boundaries_counter} boundaries in PostgreSql  in #{display_time(Time.now - start)} seconds"
end

#backup_ways ⇒ Object

def update_node_with_way(way_id, node_ids)

# Update node data with way id
node_ids.each do |node_id|
  node = Marshal.load(nodes_database[node_id])
  node.add_way(way_id)
  node.end_of_way = true if [node_ids.first, node_ids.last].include?(node_id)
  nodes_database[node_id] = Marshal.dump(node)
end

end



204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 204

def backup_ways
  Rails.logger.info "Begin to backup ways in LevelDB"
  start = Time.now
  ways_parser = ::PbfParser.new(pbf_file)
  ways_counter = 0 
  
  # Process the file until it finds any way.
  ways_parser.next until ways_parser.ways.any?
  
  # Once it found at least one way, iterate to find the remaining ways.     
  until ways_parser.ways.empty?
    ways_database.batch do |batch|
      ways_parser.ways.each do |way|            
        way_id = way[:id].to_s
        
        if way.key?(:tags) && required_way?(@@way_required_tags_keys, way[:tags])
          select_tags = selected_tags(way[:tags], @@way_selected_tags_keys)
          opt_tags = selected_tags(way[:tags], @@way_optionnal_tags_keys)
          node_ids = way.key?(:refs) ? way[:refs].collect(&:to_s) : []

          way = Way.new( way_id, node_ids, car?(opt_tags), bike?(opt_tags), train?(opt_tags), pedestrian?(opt_tags), select_tags["name"], select_tags["maxspeed"], select_tags["oneway"], select_tags["boundary"], select_tags["admin_level"], select_tags["addr:housenumber"], opt_tags )

          ways_splitted = (way.boundary.present? || way.addr_housenumber.present?) ? [way] : split_way_with_nodes(way) # Don't split boundary and adress way               
          
          ways_splitted.each do |way_splitted|
            ways_counter+= 1
            batch[ way_splitted.id ] = Marshal.dump( way_splitted )        
          end
        end
      end
    end        
    
    # When there's no more fileblocks to parse, #next returns false
    # This avoids an infinit loop when the last fileblock still contains ways
    break unless ways_parser.next        
  end

  Rails.logger.info "Finish to backup #{ways_counter} ways in LevelDB  in #{display_time(Time.now - start)} seconds"
end

#close_junctions_database ⇒ Object 



50
51
52
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 50

def close_junctions_database
  junctions_database.close!
end

#close_nodes_database ⇒ Object 



26
27
28
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 26

def close_nodes_database
  nodes_database.close!
end

#close_physical_roads_database ⇒ Object 



62
63
64
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 62

def close_physical_roads_database
  physical_roads_database.close!
end

#close_ways_database ⇒ Object 



38
39
40
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 38

def close_ways_database
  ways_database.close!
end

#delete_junctions_database ⇒ Object 



54
55
56
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 54

def delete_junctions_database
  FileUtils.remove_entry junctions_database_path if File.exists?(junctions_database_path)
end

#delete_nodes_database ⇒ Object 



30
31
32
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 30

def delete_nodes_database
  FileUtils.remove_entry nodes_database_path if File.exists?(nodes_database_path)
end

#delete_physical_roads_database ⇒ Object 



66
67
68
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 66

def delete_physical_roads_database
  FileUtils.remove_entry physical_roads_database_path if File.exists?(physical_roads_database_path)
end

#delete_ways_database ⇒ Object 



42
43
44
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 42

def delete_ways_database
  FileUtils.remove_entry ways_database_path if File.exists?(ways_database_path)
end

#display_time(time_difference) ⇒ Object 



70
71
72
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 70

def display_time(time_difference)
  Time.at(time_difference.to_i).utc.strftime "%H:%M:%S"
end

#find_boundary(way_geometry) ⇒ Object 



668
669
670
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 668

def find_boundary(way_geometry)
  ActiveRoad::Boundary.first_contains(way_geometry)
end

#import ⇒ Object 



74
75
76
77
78
79
80
81
82
83
84
85
86
87
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 74

def import
  delete_nodes_database
  delete_ways_database
  delete_junctions_database
  delete_physical_roads_database

  leveldb_import
  postgres_import
  
  close_nodes_database
  close_ways_database
  close_junctions_database
  close_physical_roads_database
end

#iterate_nodes ⇒ Object 



279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 279

def iterate_nodes
  Rails.logger.debug "Begin to backup nodes in PostgreSql"

  start = Time.now
  nodes_counter = street_numbers_counter = 0
  junctions_values = []
  street_numbers_values = []    
  nodes_database_size = nodes_database.count
  
  # traverse records by iterator
  junction_columns = ["objectid", "geometry", "created_at", "updated_at"]
  street_number_columns = ["objectid", "geometry", "number", "tags", "created_at", "updated_at"]
  
  CSV.open("/tmp/junctions.csv", "wb:UTF-8") do |junctions_csv|        
    CSV.open("/tmp/street_numbers.csv", "wb:UTF-8") do |street_numbers_csv|          
      junctions_csv << junction_columns
      street_numbers_csv << street_number_columns
      
      nodes_database.each { |key, value|            
        node = Marshal.load(value)
        geometry = GeoRuby::SimpleFeatures::Point.from_x_y( node.lon, node.lat, 4326) if( node.lon && node.lat )
        
        if node.ways.present? && (node.ways.count >= 2 || node.end_of_way == true )  # Take node with at least two ways or at the end of a way
          nodes_counter += 1
          junctions_csv << [ node.id, geometry.as_hex_ewkb, Time.now, Time.now ]
        end       
        
        if node.addr_housenumber.present?
          street_numbers_counter += 1
          street_numbers_csv << [ node.id, geometry.as_hex_ewkb, node.addr_housenumber, "#{node.tags.to_s.gsub(/[{}]/, '')}", Time.now, Time.now ]
        end
          
      }
    end
  end
         
  ActiveRoad::Junction.transaction do                                         
    ActiveRoad::Junction.pg_copy_from "/tmp/junctions.csv"
  end
  
  ActiveRoad::StreetNumber.transaction do
    ActiveRoad::StreetNumber.pg_copy_from "/tmp/street_numbers.csv"
  end
  
  Rails.logger.info "Finish to backup #{nodes_counter} nodes and #{street_numbers_counter} street_numbres in PostgreSql in #{display_time(Time.now - start)} seconds"         
end

#iterate_ways ⇒ Object 



326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 326

def iterate_ways
  Rails.logger.info "Begin to backup ways in PostgreSql"
  start = Time.now
   
  ways_counter = 0
  street_numbers_counter = 0
  ways_database_size = ways_database.count

  # traverse records by iterator
  physical_road_columns = ["objectid", "car", "bike", "train", "pedestrian", "name", "geometry", "boundary_id", "tags", "created_at", "updated_at"]
  street_number_columns = ["objectid", "geometry", "number", "tags", "created_at", "updated_at"]     
  
  CSV.open("/tmp/physical_roads.csv", "wb:UTF-8") do |physical_roads_csv|
    CSV.open("/tmp/street_numbers2.csv", "wb:UTF-8") do |street_numbers_csv|          
      physical_roads_csv << physical_road_columns
      street_numbers_csv << street_number_columns
      
      ways_database.each { |key, value|          
        way = Marshal.load(value)
        
        unless way.boundary.present? # Use ways not used in relation for boundaries
          nodes = []
          way.nodes.each_with_index do |node_id, index|
            node = Marshal.load( nodes_database[node_id.to_s] )
            nodes << node                
          end
          way_geometry = way_geometry(nodes)
          
          if way.addr_housenumber.present? # If ways with adress
            street_numbers_counter += 1
            street_numbers_csv << [ way.id, way_geometry.envelope.center.as_hex_ewkb, way.addr_housenumber, "#{way.options.to_s.gsub(/[{}]/, '')}", Time.now, Time.now ]                
          else
            ways_counter += 1                      
            way_boundary = way.boundary.present? ? way.boundary.to_i : nil
            physical_roads_csv << [ way.id, way.car, way.bike, way.train, way.pedestrian, way.name, way_geometry.as_hex_ewkb, way_boundary, "#{way.options.to_s.gsub(/[{}]/, '')}", Time.now, Time.now ]
          end
        end
      }
    end
  end
  
  # Save physical roads
  ActiveRoad::PhysicalRoad.transaction do                                         
    ActiveRoad::PhysicalRoad.pg_copy_from "/tmp/physical_roads.csv"
  end

  ActiveRoad::StreetNumber.transaction do                                         
    ActiveRoad::StreetNumber.pg_copy_from "/tmp/street_numbers2.csv"        
  end

  Rails.logger.info "Finish to backup #{ways_counter} ways and #{street_numbers_counter} street numbers in PostgreSql in #{display_time(Time.now - start)} seconds"
end

#junctions_database ⇒ Object 



46
47
48
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 46

def junctions_database
  @junctions_database ||= LevelDBNative::DB.make junctions_database_path, :create_if_missing => true, :block_cache_size => 16 * 1024 * 1024
end

#leveldb_import ⇒ Object 



89
90
91
92
93
94
95
96
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 89

def leveldb_import
  # Save nodes in temporary file
  backup_nodes
  # Update nodes with ways in temporary file
  update_nodes_with_way
  # Save ways in temporary file
  backup_ways      
end

#nodes_database ⇒ Object 



22
23
24
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 22

def nodes_database
  @nodes_database ||= LevelDBNative::DB.make nodes_database_path, :create_if_missing => true, :block_cache_size => 16 * 1024 * 1024
end

#physical_roads_database ⇒ Object 



58
59
60
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 58

def physical_roads_database
  @physical_roads_database ||= LevelDBNative::DB.make physical_roads_database_path, :create_if_missing => true, :block_cache_size => 16 * 1024 * 1024
end

#postgres_import ⇒ Object 



98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 98

def postgres_import
  # Save nodes in junctions
  iterate_nodes      
  
  # Save relations in boundary
  backup_relations_pgsql if split_ways

  # Save ways in physical roads
  iterate_ways

  save_junctions_and_physical_roads_temporary
  save_physical_road_conditionnal_costs_and_junctions

  # Split and affect boundary to each way     
  split_way_with_boundaries if split_ways
  
  # Save logical roads from physical roads
  backup_logical_roads_pgsql if split_ways
end

#save_junctions_and_physical_roads_temporary ⇒ Object 



379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 379

def save_junctions_and_physical_roads_temporary
  Rails.logger.info "Begin to backup physical_roads and junctions in LevelDb"
  
  start = Time.now      
  junctions_database.batch do |batch|
    ActiveRoad::Junction.select("id,objectid").find_each do |junction|
      junctions_database[junction.objectid] = junction.id.to_s
    end
  end

  physical_roads_database.batch do |batch|
    ActiveRoad::PhysicalRoad.select("id,objectid").find_each do |physical_road|
      physical_roads_database[physical_road.objectid] = physical_road.id.to_s 
    end
  end

  Rails.logger.info "Finish to backup physical_roads and junctions in LevelDb in #{display_time(Time.now - start)} seconds"
end

#save_physical_road_conditionnal_costs_and_junctions ⇒ Object 



398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 398

def save_physical_road_conditionnal_costs_and_junctions
  Rails.logger.info "Begin to backup ways in PostgreSql"
  
  start = Time.now
  physical_road_conditionnal_costs_counter = junctions_physical_roads_counter = 0
  physical_road_conditionnal_cost_columns = ["tags", "cost", "physical_road_id"]
  junction_physical_road_columns = ["physical_road_id", "junction_id"]
  
  CSV.open("/tmp/physical_road_conditionnal_costs.csv", "wb:UTF-8") do |physical_road_conditionnal_costs_csv|
    CSV.open("/tmp/junctions_physical_roads.csv", "wb:UTF-8") do |junctions_physical_roads_csv|
      physical_road_conditionnal_costs_csv << physical_road_conditionnal_cost_columns
      junctions_physical_roads_csv << junction_physical_road_columns
      
      ways_database.each { |key, value|
        way = Marshal.load(value)

        # Save physical road conditionnal cost not for boundaries or street numbers
        unless way.boundary.present? || way.addr_housenumber.present?
          way_conditionnal_costs = physical_road_conditionnal_costs(way)
          way_conditionnal_costs.each do |way_conditionnal_cost|
            physical_road_conditionnal_costs_counter += 1
            physical_road_conditionnal_costs_csv << way_conditionnal_cost + [ physical_roads_database[way.id] ]
          end

          way.nodes.each do |node_id|
            junction_id = junctions_database[node_id]
            junctions_physical_roads_counter += 1
            junctions_physical_roads_csv << [ physical_roads_database[way.id], junction_id ] if junction_id.present?
          end
        end
      }
    end
  end
  
  # Save physical road conditionnal costs
  ActiveRoad::PhysicalRoadConditionnalCost.transaction do                                         
    ActiveRoad::PhysicalRoadConditionnalCost.pg_copy_from "/tmp/physical_road_conditionnal_costs.csv"
  end

  # Save physical road and junctions link
  ActiveRoad::JunctionsPhysicalRoad.transaction do                                         
    ActiveRoad::JunctionsPhysicalRoad.pg_copy_from "/tmp/junctions_physical_roads.csv"
  end

  Rails.logger.info "Finish to backup #{junctions_physical_roads_counter} junctions_physical_roads and #{physical_road_conditionnal_costs_counter} physical_road_conditionnal_costs in PostgreSql in #{display_time(Time.now - start)} seconds"
end

#split_way_with_boundaries ⇒ Object 



445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 445

def split_way_with_boundaries
  Rails.logger.info "Begin to split and affect boundaries to ways in PostgreSql"
  start = Time.now

  # Update physical roads entirely contains in boundaries
  ActiveRoad::PhysicalRoad.connection.select_all("SELECT physical_road.id AS physical_road_id, boundary.id AS boundary_id FROM physical_roads physical_road, boundaries boundary WHERE ST_Covers( boundary.geometry, physical_road.geometry)").each_slice(@@pg_batch_size) do |group|
    ActiveRoad::PhysicalRoad.transaction do 
      group.each do |element|
        ActiveRoad::PhysicalRoad.update(element["physical_road_id"], :boundary_id => element["boundary_id"])
      end
    end
  end

  if split_ways
    simple_ways = []
    simple_ways_not_line_string = 0

    # Fix : Produce 2 ways when way is tangent to boundary borders for each boundary
    # Get geometries in boundary      
    sql = "SELECT b.id AS boundary_id, p.id AS physical_road_id, p.objectid AS physical_road_objectid, p.tags AS physical_road_tags, ST_AsText(p.geometry) AS physical_road_geometry, 
j1.objectid AS departure_objectid, ST_AsText(j1.geometry) AS departure_geometry, 
j2.objectid AS arrival_objectid, ST_AsText(j2.geometry) AS arrival_geometry, 
ST_AsText( (ST_Dump(ST_Intersection( p.geometry , b.geometry))).geom ) AS intersection_geometry 
FROM physical_roads p, boundaries b, junctions j1, junctions j2, junctions_physical_roads jp, junctions_physical_roads jp2 
WHERE p.boundary_id IS NULL AND ST_Crosses( b.geometry, p.geometry)
AND j1.id = jp.junction_id AND p.id = jp.physical_road_id AND ST_Equals(ST_StartPoint(p.geometry), j1.geometry)
AND j2.id = jp2.junction_id AND p.id = jp2.physical_road_id AND ST_Equals(ST_EndPoint(p.geometry), j2.geometry)".gsub(/^( |\t)+/, "")      
    ActiveRoad::PhysicalRoad.connection.select_all( sql ).each do |result|
      intersection_geometry = GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['intersection_geometry']}")

      # Not take in consideration point intersection!!
      if intersection_geometry.class == GeoRuby::SimpleFeatures::LineString
        simple_way = SimpleWay.new(result["boundary_id"], result["physical_road_id"], result["physical_road_objectid"], result["physical_road_tags"], GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['physical_road_geometry']}"), result["departure_objectid"], GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['departure_geometry']}"), result["arrival_objectid"], GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['arrival_geometry']}"), intersection_geometry )
        # Delete boucle line string Ex : 9938647-4
        simple_ways << simple_way if simple_way.departure != simple_way.arrival
      else
        simple_ways_not_line_string += 1
      end
    end
    
    # Get geometries not in boundaries      
    sql = "SELECT ST_AsText( (ST_Dump(difference_geometry)).geom ) AS difference_geometry, v.id AS physical_road_id, v.objectid AS physical_road_objectid, v.tags AS physical_road_tags, ST_AsText(v.geometry) AS physical_road_geometry,
j1.objectid AS departure_objectid, ST_AsText(j1.geometry) AS departure_geometry, 
j2.objectid AS arrival_objectid, ST_AsText(j2.geometry) AS arrival_geometry
FROM 
( SELECT pr.id, pr.objectid, pr.tags, pr.geometry, pr.boundary_id, ST_Difference( pr.geometry, ST_Union( b.geometry)) as difference_geometry 
FROM physical_roads pr, boundaries b 
WHERE pr.boundary_id IS NULL AND ST_Crosses( b.geometry, pr.geometry)
GROUP BY pr.id, pr.geometry) v, 
junctions j1, junctions j2, junctions_physical_roads jp, junctions_physical_roads jp2
WHERE j1.id = jp.junction_id AND v.id = jp.physical_road_id AND ST_Equals(ST_StartPoint(v.geometry), j1.geometry)
AND j2.id = jp2.junction_id AND v.id = jp2.physical_road_id AND ST_Equals(ST_EndPoint(v.geometry), j2.geometry)
AND NOT ST_IsEmpty(difference_geometry)".gsub(/^( |\t)+/, "") 
    ActiveRoad::PhysicalRoad.connection.select_all( sql ).each do |result|
      difference_geometry = GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['difference_geometry']}")
      if difference_geometry.class == GeoRuby::SimpleFeatures::LineString
        simple_way = SimpleWay.new(nil, result["physical_road_id"], result["physical_road_objectid"], result["physical_road_tags"], GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['physical_road_geometry']}"), result["departure_objectid"], GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['departure_geometry']}"), result["arrival_objectid"], GeoRuby::SimpleFeatures::Geometry.from_ewkt("SRID=#{ActiveRoad.srid};#{result['arrival_geometry']}"), difference_geometry )
        # Delete boucle line string Ex : 9938647-4
        simple_ways << simple_way if simple_way.departure != simple_way.arrival
      else
        simple_ways_not_line_string += 1
      end
    end
   
    # Prepare reordering ways         
    simple_ways_by_old_physical_road_id = simple_ways.group_by{|sw| sw.old_physical_road_id}

    # Hack : in the code we take the first one which has an intersection point and it deletes
    # dual segment tangent on the boundary borders 
    simple_ways_by_old_physical_road_id.each do |old_physical_road_id, ways|
      ways.each do |way|
        if way.departure == way.old_departure_geometry
          way.departure_objectid = way.old_departure_objectid
          way.previous = nil
        else
          way.departure_objectid = way.default_departure_objectid
          way.previous = ways.detect{ |select_way| select_way.arrival == way.departure }
        end
        
        if way.arrival == way.old_arrival_geometry
          way.arrival_objectid = way.old_arrival_objectid
          way.next = nil 
        else
          way.arrival_objectid = way.default_arrival_objectid
          way.next = ways.detect{ |select_way| select_way.departure == way.arrival }
        end          
      end
    end       

    # Save new ways and junctions
    #physical_roads ||= ActiveRoad::PhysicalRoad.where(:objectid => simple_ways_by_old_physical_road_id.keys).includes(:conditionnal_costs)
    
    simple_ways_by_old_physical_road_id.each_slice(1000) { |group|
      ActiveRoad::PhysicalRoad.transaction do            
        
        group.each do |old_physical_road_id, ways|
          #puts ways.sort.inspect             
          next_way = ways.detect{ |select_way| select_way.previous == nil }
          way_counter = 0
          junction_counter = 0

          
          while next_way != nil
            start = Time.now

            #old_physical_road = physical_roads.where(:id => old_physical_road_id)               
            #physical_road.conditionnal_costs = old_physical_road.conditionnal_costs

            # Create departure
            if next_way.previous != nil
              departure = ActiveRoad::Junction.where(:objectid => "#{next_way.departure_objectid}-#{junction_counter}").first_or_create( :geometry => next_way.departure )
              junction_counter += 1
            else
              departure = ActiveRoad::Junction.find_by_objectid(next_way.departure_objectid) 
            end
            
            # Create arrival
            if next_way.next != nil
              arrival = ActiveRoad::Junction.where(:objectid => "#{next_way.arrival_objectid}-#{junction_counter}").first_or_create( :geometry => next_way.arrival )
            else
              arrival = ActiveRoad::Junction.find_by_objectid(next_way.arrival_objectid)
            end

            old_physical_road_tags = next_way.old_physical_road_tags_hash
            old_physical_road_tags["first_node_id"] = departure.objectid
            old_physical_road_tags["last_node_id"] =  arrival.objectid 
            
            physical_road = ActiveRoad::PhysicalRoad.create! :objectid => "#{next_way.old_physical_road_objectid}-#{way_counter}", :boundary_id => next_way.boundary_id, :geometry => next_way.geometry, :tags => old_physical_road_tags
            
            # Add departure and arrival to physical road
            physical_road.junctions << [departure, arrival]

            way_counter += 1

            if way_counter > ways.size
              Rails.logger.error "Infinite boucle when save physical road splitted with boundaries"
              raise Exception.new "Infinite boucle when save physical road splitted with boundaries"
            end
            
            next_way = next_way.next                                
          end
          
        end
      end
    }
    
    # Delete old ways
    ActiveRoad::PhysicalRoad.destroy(simple_ways_by_old_physical_road_id.keys)
  end
  
  Rails.logger.info "Finish to split and affect boundaries to ways in PostgreSql in #{display_time(Time.now - start)} seconds"
end

#split_way_with_nodes(way) ⇒ Object 



244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 244

def split_way_with_nodes(way)
  nodes_used = []
  nodes = []
  # Get nodes really used and all nodes (used and for geometry need) for a way
  way.nodes.each_with_index do |node_id, index|
    node = Marshal.load( nodes_database[node_id.to_s] )
    nodes << node
    nodes_used << index if node.used?
  end

  ways_nodes = []
  # Split way between each nodes used
  if split_ways
    nodes_used.each_with_index do |before_node, index|        
      ways_nodes << nodes.values_at(before_node..nodes_used[ index + 1]) if before_node != nodes_used.last
    end
  else
    ways_nodes = [nodes]
  end

  ways_splitted = []
  ways_nodes.each_with_index do |way_nodes, index|                
    way_tags = way.options.dup         
    way_tags["first_node_id"] = way_nodes.first.id
    way_tags["last_node_id"] =  way_nodes.last.id

    # Don't add way if node_ids contains less than 2 nodes
    if way_nodes.present? && way_nodes.size > 1
      ways_splitted <<  Way.new( way.id + "-#{index}", way_nodes.collect(&:id), way.car, way.bike, way.train, way.pedestrian, way.name, way.maxspeed, way.oneway, way.boundary, way.admin_level, way.addr_housenumber, way_tags )
    end        
  end

  ways_splitted
end

#update_nodes_with_way ⇒ Object 



145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 145

def update_nodes_with_way
  Rails.logger.info "Update way in nodes in LevelDB"
  start = Time.now
  ways_parser = ::PbfParser.new(pbf_file)
  ways_counter = 0 
  
  # Process the file until it finds any way.
  ways_parser.next until ways_parser.ways.any?
  
  # Once it found at least one way, iterate to find the remaining ways.     
  until ways_parser.ways.empty?
    nodes_readed = {}
    nodes_database.batch do |batch|
      ways_parser.ways.each do |way|            
        way_id = way[:id].to_s
        
        if way.key?(:tags) && required_way?(@@way_for_physical_road_required_tags_keys, way[:tags])
          # Don't add way to nodes if a way is a boundary
          select_tags = selected_tags(way[:tags], @@way_selected_tags_keys)
          node_ids = way.key?(:refs) ? way[:refs].collect(&:to_s) : []
          
          if node_ids.present? && node_ids.size > 1
            ways_counter+= 1
            node_ids.each do |node_id|
              if nodes_readed.has_key?(node_id)                    
                node = nodes_readed[node_id]
              else
                node = Marshal.load(nodes_database[node_id])
              end
              node.add_way(way_id)
              node.end_of_way = true if [node_ids.first, node_ids.last].include?(node_id)
              nodes_readed[node_id] = node
            end
          end        
        end
      end
      nodes_readed.each_pair do |node_readed_id, node_readed|
        batch[node_readed_id] = Marshal.dump(node_readed)
      end
    end        
    
    # When there's no more fileblocks to parse, #next returns false
    # This avoids an infinit loop when the last fileblock still contains ways
    break unless ways_parser.next        
  end

  Rails.logger.info "Finish to update #{ways_counter} ways in nodes in LevelDB  in #{display_time(Time.now - start)} seconds"
end

#way_geometry(nodes) ⇒ Object 



659
660
661
662
663
664
665
666
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 659

def way_geometry(nodes)
  points = []
  nodes.each do |node|
    points << GeoRuby::SimpleFeatures::Point.from_x_y(node.lon, node.lat, 4326)
  end

  GeoRuby::SimpleFeatures::LineString.from_points(points, 4326) if points.present? &&  1 < points.count     
end

#ways_database ⇒ Object 



34
35
36
 
# File 'app/models/active_road/osm_pbf_importer_level_db.rb', line 34

def ways_database
  @ways_database ||= LevelDBNative::DB.make ways_database_path, :create_if_missing => true, :block_cache_size => 16 * 1024 * 1024
end