Libosmium  2.11.0
Fast and flexible C++ library for working with OpenStreetMap data
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
collector.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_RELATIONS_COLLECTOR_HPP
2 #define OSMIUM_RELATIONS_COLLECTOR_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2017 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <algorithm>
37 #include <cassert>
38 #include <cstddef>
39 #include <cstdint>
40 #include <functional>
41 #include <iomanip>
42 #include <iostream>
43 #include <utility>
44 #include <vector>
45 
46 #include <osmium/osm/item_type.hpp>
47 #include <osmium/osm/object.hpp>
48 #include <osmium/osm/relation.hpp>
49 #include <osmium/osm/types.hpp>
50 #include <osmium/handler.hpp>
51 #include <osmium/memory/buffer.hpp>
52 #include <osmium/util/iterator.hpp>
53 #include <osmium/visitor.hpp>
54 
55 #include <osmium/relations/detail/relation_meta.hpp>
56 #include <osmium/relations/detail/member_meta.hpp>
57 
58 namespace osmium {
59 
60  class Node;
61  class Way;
62 
66  namespace relations {
67 
96  template <typename TCollector, bool TNodes, bool TWays, bool TRelations>
97  class Collector {
98 
103 
104  TCollector& m_collector;
105 
106  public:
107 
108  explicit HandlerPass1(TCollector& collector) noexcept :
109  m_collector(collector) {
110  }
111 
113  if (m_collector.keep_relation(relation)) {
114  m_collector.add_relation(relation);
115  }
116  }
117 
118  }; // class HandlerPass1
119 
120  public:
121 
126 
127  TCollector& m_collector;
128 
129  public:
130 
131  explicit HandlerPass2(TCollector& collector) noexcept :
132  m_collector(collector) {
133  }
134 
135  void node(const osmium::Node& node) {
136  if (TNodes) {
137  if (! m_collector.find_and_add_object(node)) {
138  m_collector.node_not_in_any_relation(node);
139  }
140  }
141  }
142 
143  void way(const osmium::Way& way) {
144  if (TWays) {
145  if (! m_collector.find_and_add_object(way)) {
146  m_collector.way_not_in_any_relation(way);
147  }
148  }
149  }
150 
152  if (TRelations) {
153  if (! m_collector.find_and_add_object(relation)) {
154  m_collector.relation_not_in_any_relation(relation);
155  }
156  }
157  }
158 
159  void flush() {
160  m_collector.flush();
161  }
162 
163  }; // class HandlerPass2
164 
165  private:
166 
167  HandlerPass2 m_handler_pass2;
168 
169  // All relations we are interested in will be kept in this buffer
171 
172  // All members we are interested in will be kept in this buffer
174 
176  std::vector<RelationMeta> m_relations;
177 
182  using mm_vector_type = std::vector<MemberMeta>;
183  using mm_iterator = mm_vector_type::iterator;
185 
187 
188  using callback_func_type = std::function<void(osmium::memory::Buffer&&)>;
190 
191  static constexpr size_t initial_buffer_size = 1024 * 1024;
192 
194  auto& mmv = member_meta(type);
195  return make_range(std::equal_range(mmv.begin(), mmv.end(), MemberMeta(id)));
196  }
197 
198  public:
199 
204  m_handler_pass2(*static_cast<TCollector*>(this)),
205  m_relations_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
206  m_members_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
207  m_relations(),
208  m_member_meta() {
209  }
210 
211  protected:
212 
213  std::vector<MemberMeta>& member_meta(const item_type type) {
214  return m_member_meta[static_cast<uint16_t>(type) - 1];
215  }
216 
218  return m_callback;
219  }
220 
221  const std::vector<RelationMeta>& relations() const {
222  return m_relations;
223  }
224 
234  bool keep_relation(const osmium::Relation& /*relation*/) const {
235  return true;
236  }
237 
248  bool keep_member(const osmium::relations::RelationMeta& /*relation_meta*/, const osmium::RelationMember& /*member*/) const {
249  return true;
250  }
251 
259  void node_not_in_any_relation(const osmium::Node& /*node*/) {
260  }
261 
269  void way_not_in_any_relation(const osmium::Way& /*way*/) {
270  }
271 
279  void relation_not_in_any_relation(const osmium::Relation& /*relation*/) {
280  }
281 
293  void flush() {
294  }
295 
301  m_relations.erase(
302  std::remove_if(m_relations.begin(), m_relations.end(), has_all_members()),
303  m_relations.end()
304  );
305  }
306 
307  const osmium::Relation& get_relation(size_t offset) const {
308  assert(m_relations_buffer.committed() > offset);
309  return m_relations_buffer.get<osmium::Relation>(offset);
310  }
311 
315  const osmium::Relation& get_relation(const RelationMeta& relation_meta) const {
316  return get_relation(relation_meta.relation_offset());
317  }
318 
322  const osmium::Relation& get_relation(const MemberMeta& member_meta) const {
323  return get_relation(m_relations[member_meta.relation_pos()]);
324  }
325 
326  osmium::OSMObject& get_member(size_t offset) const {
327  assert(m_members_buffer.committed() > offset);
328  return m_members_buffer.get<osmium::OSMObject>(offset);
329  }
330 
331  private:
332 
341  void add_relation(const osmium::Relation& relation) {
342  const size_t offset = m_relations_buffer.committed();
343  m_relations_buffer.add_item(relation);
344 
345  RelationMeta relation_meta(offset);
346 
347  int n = 0;
348  for (auto& member : m_relations_buffer.get<osmium::Relation>(offset).members()) {
349  if (static_cast<TCollector*>(this)->keep_member(relation_meta, member)) {
350  member_meta(member.type()).emplace_back(member.ref(), m_relations.size(), n);
351  relation_meta.increment_need_members();
352  } else {
353  member.set_ref(0); // set member id to zero to indicate we are not interested
354  }
355  ++n;
356  }
357 
358  assert(offset == m_relations_buffer.committed());
359  if (relation_meta.has_all_members()) {
360  m_relations_buffer.rollback();
361  } else {
362  m_relations_buffer.commit();
363  m_relations.push_back(std::move(relation_meta));
364  }
365  }
366 
372  std::sort(m_member_meta[0].begin(), m_member_meta[0].end());
373  std::sort(m_member_meta[1].begin(), m_member_meta[1].end());
374  std::sort(m_member_meta[2].begin(), m_member_meta[2].end());
375  }
376 
378  return std::count_if(range.begin(), range.end(), [](MemberMeta& mm) {
379  return !mm.removed();
380  });
381  }
382 
391  auto range = find_member_meta(object.type(), object.id());
392 
393  if (count_not_removed(range) == 0) {
394  // nothing found
395  return false;
396  }
397 
398  {
399  members_buffer().add_item(object);
400  const size_t member_offset = members_buffer().commit();
401 
402  for (auto& member_meta : range) {
403  member_meta.set_buffer_offset(member_offset);
404  }
405  }
406 
407  for (auto& member_meta : range) {
408  if (member_meta.removed()) {
409  break;
410  }
411  assert(member_meta.member_id() == object.id());
412  assert(member_meta.relation_pos() < m_relations.size());
413  RelationMeta& relation_meta = m_relations[member_meta.relation_pos()];
414  assert(member_meta.member_pos() < get_relation(relation_meta).members().size());
415  relation_meta.got_one_member();
416  if (relation_meta.has_all_members()) {
417  const size_t relation_offset = member_meta.relation_pos();
418  static_cast<TCollector*>(this)->complete_relation(relation_meta);
419  clear_member_metas(relation_meta);
420  m_relations[relation_offset] = RelationMeta();
422  }
423  }
424 
425  return true;
426  }
427 
428  void clear_member_metas(const osmium::relations::RelationMeta& relation_meta) {
429  const osmium::Relation& relation = get_relation(relation_meta);
430  for (const auto& member : relation.members()) {
431  if (member.ref() != 0) {
432  const auto range = find_member_meta(member.type(), member.ref());
433  assert(!range.empty());
434 
435  // if this is the last time this object was needed
436  // then mark it as removed
437  if (count_not_removed(range) == 1) {
438  get_member(range.begin()->buffer_offset()).set_removed(true);
439  }
440 
441  for (auto& member_meta : range) {
442  if (!member_meta.removed() && relation.id() == get_relation(member_meta).id()) {
443  member_meta.remove();
444  break;
445  }
446  }
447  }
448  }
449  }
450 
451  public:
452 
453  uint64_t used_memory() const {
454  const uint64_t nmembers = m_member_meta[0].capacity() + m_member_meta[1].capacity() + m_member_meta[2].capacity();
455  const uint64_t members = nmembers * sizeof(MemberMeta);
456  const uint64_t relations = m_relations.capacity() * sizeof(RelationMeta);
457  const uint64_t relations_buffer_capacity = m_relations_buffer.capacity();
458  const uint64_t members_buffer_capacity = m_members_buffer.capacity();
459 
460  std::cerr << " nR = m_relations.capacity() ........... = " << std::setw(12) << m_relations.capacity() << "\n";
461  std::cerr << " nMN = m_member_meta[NODE].capacity() ... = " << std::setw(12) << m_member_meta[0].capacity() << "\n";
462  std::cerr << " nMW = m_member_meta[WAY].capacity() .... = " << std::setw(12) << m_member_meta[1].capacity() << "\n";
463  std::cerr << " nMR = m_member_meta[RELATION].capacity() = " << std::setw(12) << m_member_meta[2].capacity() << "\n";
464  std::cerr << " nM = m_member_meta[*].capacity() ...... = " << std::setw(12) << nmembers << "\n";
465 
466  std::cerr << " sRM = sizeof(RelationMeta) ............. = " << std::setw(12) << sizeof(RelationMeta) << "\n";
467  std::cerr << " sMM = sizeof(MemberMeta) ............... = " << std::setw(12) << sizeof(MemberMeta) << "\n\n";
468 
469  std::cerr << " nR * sRM ............................... = " << std::setw(12) << relations << "\n";
470  std::cerr << " nM * sMM ............................... = " << std::setw(12) << members << "\n";
471  std::cerr << " relations_buffer_capacity .............. = " << std::setw(12) << relations_buffer_capacity << "\n";
472  std::cerr << " members_buffer_capacity ................ = " << std::setw(12) << members_buffer_capacity << "\n";
473 
474  const uint64_t total = relations + members + relations_buffer_capacity + members_buffer_capacity;
475 
476  std::cerr << " total .................................. = " << std::setw(12) << total << "\n";
477  std::cerr << " =======================================================\n";
478 
479  return relations_buffer_capacity + members_buffer_capacity + relations + members;
480  }
481 
485  HandlerPass2& handler(const callback_func_type& callback = nullptr) {
486  m_callback = callback;
487  return m_handler_pass2;
488  }
489 
491  return m_members_buffer;
492  }
493 
506  const auto range = find_member_meta(type, id);
507  assert(!range.empty());
508  return range.begin()->is_available();
509  }
510 
521  const auto range = find_member_meta(type, id);
522  assert(!range.empty());
523  assert(range.begin()->is_available());
524  return range.begin()->buffer_offset();
525  }
526 
544  const auto range = find_member_meta(type, id);
545  assert(!range.empty());
546  if (range.begin()->is_available()) {
547  return std::make_pair(true, range.begin()->buffer_offset());
548  } else {
549  return std::make_pair(false, 0);
550  }
551  }
552 
553  template <typename TIter>
554  void read_relations(TIter begin, TIter end) {
555  HandlerPass1 handler(*static_cast<TCollector*>(this));
556  osmium::apply(begin, end, handler);
558  }
559 
560  template <typename TSource>
561  void read_relations(TSource& source) {
562  read_relations(std::begin(source), std::end(source));
563  source.close();
564  }
565 
566  void moving_in_buffer(size_t old_offset, size_t new_offset) {
567  const osmium::OSMObject& object = m_members_buffer.get<osmium::OSMObject>(old_offset);
568  auto range = find_member_meta(object.type(), object.id());
569  for (auto& member_meta : range) {
570  assert(member_meta.buffer_offset() == old_offset);
571  member_meta.set_buffer_offset(new_offset);
572  }
573  }
574 
583  if (m_count_complete > 10000) { // XXX
584 // const size_t size_before = m_members_buffer.committed();
585  m_members_buffer.purge_removed(this);
586 /*
587  const size_t size_after = m_members_buffer.committed();
588  double percent = static_cast<double>(size_before - size_after);
589  percent /= size_before;
590  percent *= 100;
591  std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast<int>(percent) << "%)\n";
592 */
593  m_count_complete = 0;
594  }
595  }
596 
605  std::vector<const osmium::Relation*> get_incomplete_relations() const {
606  std::vector<const osmium::Relation*> relations;
607  for (const auto& relation_meta : m_relations) {
608  if (!relation_meta.has_all_members()) {
609  relations.push_back(&get_relation(relation_meta));
610  }
611  }
612  return relations;
613  }
614 
615  }; // class Collector
616 
617  } // namespace relations
618 
619 } // namespace osmium
620 
621 #endif // OSMIUM_RELATIONS_COLLECTOR_HPP
std::vector< const osmium::Relation * > get_incomplete_relations() const
Definition: collector.hpp:605
void relation(const osmium::Relation &relation)
Definition: collector.hpp:151
void clear_member_metas(const osmium::relations::RelationMeta &relation_meta)
Definition: collector.hpp:428
Definition: iterator.hpp:43
callback_func_type m_callback
Definition: collector.hpp:189
osmium::memory::Buffer & members_buffer()
Definition: collector.hpp:490
type
Definition: entity_bits.hpp:63
iterator_range< It > make_range(P &&p)
Definition: iterator.hpp:77
RelationMemberList & members()
Definition: relation.hpp:185
bool is_available(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:505
mm_vector_type m_member_meta[3]
Definition: collector.hpp:184
void way(const osmium::Way &way)
Definition: collector.hpp:143
item_type
Definition: item_type.hpp:43
void clean_assembled_relations()
Definition: collector.hpp:300
void moving_in_buffer(size_t old_offset, size_t new_offset)
Definition: collector.hpp:566
Definition: relation.hpp:168
static constexpr size_t initial_buffer_size
Definition: collector.hpp:191
Definition: handler.hpp:71
bool keep_member(const osmium::relations::RelationMeta &, const osmium::RelationMember &) const
Definition: collector.hpp:248
size_type size() const noexcept
Definition: collection.hpp:152
osmium::memory::Buffer m_members_buffer
Definition: collector.hpp:173
const std::vector< RelationMeta > & relations() const
Definition: collector.hpp:221
void read_relations(TSource &source)
Definition: collector.hpp:561
size_t get_offset(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:520
void relation_not_in_any_relation(const osmium::Relation &)
Definition: collector.hpp:279
Definition: way.hpp:72
std::vector< MemberMeta > & member_meta(const item_type type)
Definition: collector.hpp:213
bool find_and_add_object(const osmium::OSMObject &object)
Definition: collector.hpp:390
Definition: collector.hpp:125
HandlerPass2(TCollector &collector) noexcept
Definition: collector.hpp:131
std::vector< RelationMeta > m_relations
Vector with all relations we are interested in.
Definition: collector.hpp:176
std::pair< bool, size_t > get_availability_and_offset(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:543
void sort_member_meta()
Definition: collector.hpp:371
void node_not_in_any_relation(const osmium::Node &)
Definition: collector.hpp:259
std::function< void(osmium::memory::Buffer &&)> callback_func_type
Definition: collector.hpp:188
Definition: relation.hpp:57
TCollector & m_collector
Definition: collector.hpp:104
Namespace for everything in the Osmium library.
Definition: assembler.hpp:73
T & add_item(const T &item)
Definition: buffer.hpp:482
Definition: collector.hpp:97
iterator_range< mm_iterator > find_member_meta(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:193
const osmium::Relation & get_relation(size_t offset) const
Definition: collector.hpp:307
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:730
HandlerPass2 m_handler_pass2
Definition: collector.hpp:167
Collector()
Definition: collector.hpp:203
osmium::OSMObject & get_member(size_t offset) const
Definition: collector.hpp:326
size_t capacity() const noexcept
Definition: buffer.hpp:251
void relation(const osmium::Relation &relation)
Definition: collector.hpp:112
osmium::io::InputIterator< osmium::io::Reader > end(osmium::io::Reader &)
Definition: reader_iterator.hpp:45
Definition: collector.hpp:102
int64_t object_id_type
Type for OSM object (node, way, or relation) IDs.
Definition: types.hpp:45
void flush()
Definition: collector.hpp:159
std::vector< MemberMeta > mm_vector_type
Definition: collector.hpp:182
osmium::memory::Buffer m_relations_buffer
Definition: collector.hpp:170
object_id_type id() const noexcept
Get ID of this object.
Definition: object.hpp:126
uint64_t used_memory() const
Definition: collector.hpp:453
size_t committed() const noexcept
Definition: buffer.hpp:259
void apply(TIterator it, TIterator end, THandlers &&...handlers)
Definition: visitor.hpp:218
callback_func_type callback()
Definition: collector.hpp:217
Definition: buffer.hpp:97
T & get(const size_t offset) const
Definition: buffer.hpp:404
void add_relation(const osmium::Relation &relation)
Definition: collector.hpp:341
HandlerPass2 & handler(const callback_func_type &callback=nullptr)
Definition: collector.hpp:485
void flush()
Definition: collector.hpp:293
Definition: node.hpp:48
void way_not_in_any_relation(const osmium::Way &)
Definition: collector.hpp:269
int m_count_complete
Definition: collector.hpp:186
void node(const osmium::Node &node)
Definition: collector.hpp:135
void possibly_purge_removed_members()
Definition: collector.hpp:581
bool keep_relation(const osmium::Relation &) const
Definition: collector.hpp:234
const osmium::Relation & get_relation(const MemberMeta &member_meta) const
Definition: collector.hpp:322
It begin() const
Definition: iterator.hpp:59
TCollector & m_collector
Definition: collector.hpp:127
void read_relations(TIter begin, TIter end)
Definition: collector.hpp:554
osmium::io::InputIterator< osmium::io::Reader > begin(osmium::io::Reader &reader)
Definition: reader_iterator.hpp:41
HandlerPass1(TCollector &collector) noexcept
Definition: collector.hpp:108
void rollback()
Definition: buffer.hpp:370
const osmium::Relation & get_relation(const RelationMeta &relation_meta) const
Definition: collector.hpp:315
static iterator_range< mm_iterator >::iterator::difference_type count_not_removed(const iterator_range< mm_iterator > &range)
Definition: collector.hpp:377
It end() const
Definition: iterator.hpp:63
Definition: object.hpp:64
size_t commit()
Definition: buffer.hpp:354