dongho/maps
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
Dongho Kim
2025-12-15 23:10:38 +09:00
parent 169997eecd
commit 45807e3a90
19 changed files with 3438 additions and 2256 deletions
Download Patch File Download Diff File Expand all files Collapse all files

309
importer/src/main.rs
View File

@@ -11,6 +11,125 @@ use memmap2::Mmap;
const ZOOM_LEVELS: [u32; 6] = [2, 4, 6, 9, 12, 14];
// Store way geometries for multipolygon assembly
struct WayStore {
ways: HashMap<i64, Vec<i64>>, // way_id -> node_id list
}
impl WayStore {
fn new() -> Self {
Self { ways: HashMap::new() }
}
fn insert(&mut self, way_id: i64, node_refs: Vec<i64>) {
self.ways.insert(way_id, node_refs);
}
fn get(&self, way_id: i64) -> Option<&Vec<i64>> {
self.ways.get(&way_id)
}
}
// Assemble ways into MULTIPLE rings (connect end-to-end)
// Rivers like the Isar have multiple separate channels/rings
fn assemble_rings(way_ids: &[i64], way_store: &WayStore) -> Vec<Vec<i64>> {
if way_ids.is_empty() { return Vec::new(); }
// Get all way geometries
let mut segments: Vec<Vec<i64>> = Vec::new();
for &way_id in way_ids {
if let Some(nodes) = way_store.get(way_id) {
if nodes.len() >= 2 {
segments.push(nodes.clone());
}
}
}
if segments.is_empty() { return Vec::new(); }
let mut completed_rings: Vec<Vec<i64>> = Vec::new();
// Keep assembling rings until we run out of segments
while !segments.is_empty() {
// Start a new ring with the first available segment
let mut ring = segments.remove(0);
// Try to extend this ring
let max_iterations = segments.len() * segments.len() + 100;
let mut iterations = 0;
loop {
iterations += 1;
if iterations > max_iterations { break; }
let mut connected = false;
for i in 0..segments.len() {
let seg = &segments[i];
if seg.is_empty() { continue; }
let ring_start = *ring.first().unwrap();
let ring_end = *ring.last().unwrap();
let seg_start = *seg.first().unwrap();
let seg_end = *seg.last().unwrap();
if ring_end == seg_start {
// Connect: ring + seg (skip first node of seg)
ring.extend(seg[1..].iter().cloned());
segments.remove(i);
connected = true;
break;
} else if ring_end == seg_end {
// Connect: ring + reversed seg
let reversed: Vec<i64> = seg.iter().rev().cloned().collect();
ring.extend(reversed[1..].iter().cloned());
segments.remove(i);
connected = true;
break;
} else if ring_start == seg_end {
// Connect: seg + ring
let mut new_ring = seg.clone();
new_ring.extend(ring[1..].iter().cloned());
ring = new_ring;
segments.remove(i);
connected = true;
break;
} else if ring_start == seg_start {
// Connect: reversed seg + ring
let mut reversed: Vec<i64> = seg.iter().rev().cloned().collect();
reversed.extend(ring[1..].iter().cloned());
ring = reversed;
segments.remove(i);
connected = true;
break;
}
}
// Check if ring is now closed
if ring.len() >= 4 && ring.first() == ring.last() {
completed_rings.push(ring);
break; // Move to next ring
}
// If no connection was made and ring isn't closed,
// we can't extend this ring anymore
if !connected {
// Still save partial rings if they have enough points
// This helps with incomplete data - at least show something
if ring.len() >= 4 {
// Force-close the ring
let first = ring[0];
ring.push(first);
completed_rings.push(ring);
}
break;
}
}
}
completed_rings
}
struct NodeStore {
writer: Option<BufWriter<File>>,
mmap: Option<Mmap>,
@@ -196,22 +315,22 @@ fn should_include(tags: &HashMap<String, String>, zoom: u32) -> bool {
// Add Towns.
// Limited nature.
matches!(highway, Some("motorway" | "trunk" | "primary")) ||
matches!(place, Some("city" | "town" | "sea" | "ocean")) ||
matches!(place, Some("city" | "town" | "sea" | "ocean" | "island" | "islet")) || // Islands!
matches!(railway, Some("rail")) ||
matches!(natural, Some("water" | "wood" | "bay" | "strait")) ||
matches!(tags.get("landuse").map(|s| s.as_str()), Some("forest")) ||
matches!(tags.get("leisure").map(|s| s.as_str()), Some("park")) ||
matches!(waterway, Some("river" | "riverbank"))
matches!(natural, Some("water" | "wood" | "scrub" | "bay" | "strait" | "wetland" | "heath" | "sand" | "beach" | "shingle" | "bare_rock")) || // Sand/Beaches!
matches!(tags.get("landuse").map(|s| s.as_str()), Some("forest" | "grass" | "meadow" | "farmland" | "residential" | "basin" | "reservoir" | "allotments")) ||
matches!(tags.get("leisure").map(|s| s.as_str()), Some("park" | "nature_reserve" | "garden")) || // Gardens
matches!(waterway, Some("river" | "riverbank" | "canal")) // Added canal
},
12 => {
matches!(highway, Some("motorway" | "trunk" | "primary" | "secondary" | "tertiary")) ||
matches!(highway, Some("motorway" | "trunk" | "primary" | "secondary" | "tertiary" | "residential" | "unclassified" | "pedestrian" | "service" | "track")) || // Added minor roads
matches!(place, Some("city" | "town" | "village")) ||
matches!(railway, Some("rail")) ||
tags.contains_key("building") ||
tags.contains_key("landuse") ||
tags.contains_key("leisure") ||
matches!(natural, Some("water" | "wood" | "scrub" | "wetland" | "heath" | "bay" | "strait")) ||
matches!(waterway, Some("river" | "riverbank" | "stream"))
matches!(natural, Some("water" | "wood" | "scrub" | "wetland" | "heath" | "bay" | "strait" | "sand" | "beach" | "bare_rock")) ||
matches!(waterway, Some("river" | "riverbank" | "stream" | "canal" | "drain" | "ditch")) // Added canal/drain/ditch
},
_ => false
}
@@ -249,6 +368,15 @@ async fn main() -> Result<()> {
session.query("CREATE TABLE IF NOT EXISTS map_data.railways (zoom int, tile_x int, tile_y int, id bigint, tags map<text, text>, points blob, PRIMARY KEY ((zoom, tile_x, tile_y), id))", &[]).await?;
// Prepare statements
println!("Truncating tables...");
session.query("TRUNCATE map_data.nodes", &[]).await?;
session.query("TRUNCATE map_data.ways", &[]).await?;
session.query("TRUNCATE map_data.buildings", &[]).await?;
session.query("TRUNCATE map_data.water", &[]).await?;
session.query("TRUNCATE map_data.landuse", &[]).await?;
session.query("TRUNCATE map_data.railways", &[]).await?;
println!("Tables truncated.");
println!("Preparing statements...");
let insert_node = session.prepare("INSERT INTO map_data.nodes (zoom, tile_x, tile_y, id, lat, lon, tags) VALUES (?, ?, ?, ?, ?, ?, ?)").await?;
let insert_ways = session.prepare("INSERT INTO map_data.ways (zoom, tile_x, tile_y, id, tags, points) VALUES (?, ?, ?, ?, ?, ?)").await?;
@@ -337,11 +465,16 @@ async fn main() -> Result<()> {
// Run the PBF reader in a blocking task to allow blocking_send
let tx_clone = tx.clone();
let reader_handle = tokio::task::spawn_blocking(move || -> Result<(usize, usize)> {
let reader_handle = tokio::task::spawn_blocking(move || -> Result<(usize, usize, usize)> {
let tx = tx_clone;
let mut node_count = 0;
let mut way_count = 0;
let mut relation_count = 0;
let mut ways_pending = false;
let mut relations_pending = false;
// Store way geometries for multipolygon assembly
let mut way_store = WayStore::new();
// We process sequentially: Nodes first, then Ways.
// osmpbf yields nodes then ways.
@@ -399,21 +532,32 @@ async fn main() -> Result<()> {
}
way_count += 1;
// Store ALL way node refs for potential multipolygon use
let node_refs: Vec<i64> = way.refs().collect();
way_store.insert(way.id(), node_refs.clone());
let tags: HashMap<String, String> = way.tags().map(|(k, v)| (k.to_string(), v.to_string())).collect();
// Filter for highways/roads OR buildings OR landuse OR water OR railways
// Filter for highways/roads OR buildings OR landuse OR water OR railways
let is_highway = tags.contains_key("highway");
let is_building = tags.contains_key("building");
let is_water = tags.get("natural").map(|v| v == "water" || v == "wetland" || v == "bay" || v == "strait").unwrap_or(false) ||
// Split Water into Area (Polygon) and Line (Way)
let is_water_area = tags.get("natural").map(|v| v == "water" || v == "wetland" || v == "bay" || v == "strait").unwrap_or(false) ||
tags.get("place").map(|v| v == "sea" || v == "ocean").unwrap_or(false) ||
tags.get("waterway").map(|v| v == "riverbank" || v == "stream" || v == "river").unwrap_or(false) ||
tags.get("waterway").map(|v| v == "riverbank" || v == "dock").unwrap_or(false) ||
tags.get("landuse").map(|v| v == "basin" || v == "reservoir").unwrap_or(false);
let is_water_line = tags.get("waterway").map(|v| v == "stream" || v == "river" || v == "canal" || v == "drain" || v == "ditch").unwrap_or(false);
let is_landuse = tags.contains_key("leisure") ||
tags.contains_key("landuse") ||
tags.get("natural").map(|v| v == "wood" || v == "scrub" || v == "heath" || v == "wetland").unwrap_or(false);
let is_railway = tags.contains_key("railway");
if is_highway || is_building || is_water || is_landuse || is_railway {
if is_highway || is_building || is_water_area || is_water_line || is_landuse || is_railway {
let mut points = Vec::new();
// Resolve nodes from store
@@ -426,9 +570,24 @@ async fn main() -> Result<()> {
if points.len() >= 2 {
let id = way.id();
// Insert into the tile of the first point
let (first_lat, first_lon) = points[0];
let is_closed = points.first() == points.last();
// Detect if we should treat this as an area
let mut treat_as_water_area = is_water_area && is_closed;
let mut treat_as_landuse = is_landuse && is_closed;
let mut treat_as_building = is_building && is_closed;
// Fallback: If water is open (e.g. riverbank segment), treat as line
let mut treat_as_water_line = is_water_line || (is_water_area && !is_closed);
// If landuse/building is open, we skip it to avoid artifacts (giant triangles)
if (is_landuse || is_building) && !is_closed {
return;
}
for &zoom in &ZOOM_LEVELS {
if !should_include(&tags, zoom) { continue; }
@@ -442,11 +601,19 @@ async fn main() -> Result<()> {
_ => 0.0,
};
let epsilon = if is_water || is_landuse || is_highway {
if zoom <= 4 && is_landuse {
0.0 // Disable simplification for landuse at low zoom to prevent disappearing polygons
let epsilon = if treat_as_water_area || treat_as_landuse || is_highway || treat_as_water_line {
if zoom <= 4 && treat_as_landuse {
0.0 // Disable simplification for landuse at low zoom
} else if treat_as_water_area || treat_as_landuse {
// User requested "little more detail"
// Almost disable simplification for organic shapes
if zoom >= 9 {
0.0 // No simplification at zoom 9+
} else {
base_epsilon * 0.01 // 1% of standard simplification - high detail
}
} else {
base_epsilon * 0.5 // Preserve more detail for natural features AND roads
base_epsilon * 0.5 // Highways/Railways can handle some simplification
}
} else {
base_epsilon
@@ -469,17 +636,14 @@ async fn main() -> Result<()> {
line_blob.extend_from_slice(&(*lon as f32).to_le_bytes());
}
// Triangulate for polygon types (buildings, water, landuse)
if is_building || is_water || is_landuse {
// Close the loop if not closed
if final_points.first() != final_points.last() {
final_points.push(final_points[0]);
}
// Triangulate for polygon types
if treat_as_building || treat_as_water_area || treat_as_landuse {
// Already checked closure above
final_points = triangulate_polygon(&final_points);
}
if final_points.len() < 3 && (is_building || is_water || is_landuse) { continue; }
if simplified_points.len() < 2 && (is_highway || is_railway) { continue; }
if final_points.len() < 3 && (treat_as_building || treat_as_water_area || treat_as_landuse) { continue; }
if simplified_points.len() < 2 && (is_highway || is_railway || treat_as_water_line) { continue; }
let (first_lat, first_lon) = simplified_points[0];
let (x, y) = lat_lon_to_tile(first_lat, first_lon, zoom);
@@ -492,23 +656,23 @@ async fn main() -> Result<()> {
polygon_blob.extend_from_slice(&(*lon as f32).to_le_bytes());
}
// Use line_blob for highways/railways, polygon_blob for others
if is_highway {
// Use line_blob for highways/railways/water_lines, polygon_blob for others
if is_highway || treat_as_water_line {
let task = DbTask::Way { zoom: zoom_i32, table: "ways", id, tags: tags.clone(), points: line_blob.clone(), x, y };
let _ = tx.blocking_send(task);
}
if is_building {
if treat_as_building {
let task = DbTask::Way { zoom: zoom_i32, table: "buildings", id, tags: tags.clone(), points: polygon_blob.clone(), x, y };
let _ = tx.blocking_send(task);
}
if is_water {
if treat_as_water_area {
let task = DbTask::Way { zoom: zoom_i32, table: "water", id, tags: tags.clone(), points: polygon_blob.clone(), x, y };
let _ = tx.blocking_send(task);
}
if is_landuse {
if treat_as_landuse {
let task = DbTask::Way { zoom: zoom_i32, table: "landuse", id, tags: tags.clone(), points: polygon_blob.clone(), x, y };
let _ = tx.blocking_send(task);
}
@@ -521,20 +685,97 @@ async fn main() -> Result<()> {
}
}
}
Element::Relation(rel) => {
if !relations_pending {
println!("Switching to Relation processing...");
relations_pending = true;
}
relation_count += 1;
let tags: HashMap<String, String> = rel.tags().map(|(k, v)| (k.to_string(), v.to_string())).collect();
// Only process multipolygon relations
if tags.get("type").map(|t| t == "multipolygon").unwrap_or(false) {
// Check if it's a water or landuse multipolygon
// IMPORTANT: Rivers like the Isar are tagged waterway=river on the relation itself!
let is_water = tags.get("natural").map(|v| v == "water" || v == "wetland" || v == "bay").unwrap_or(false) ||
tags.get("waterway").map(|v| v == "riverbank" || v == "river" || v == "canal").unwrap_or(false) ||
tags.get("water").is_some() || // Also check water=* tag
tags.get("landuse").map(|v| v == "basin" || v == "reservoir").unwrap_or(false);
let is_landuse = tags.get("landuse").is_some() ||
tags.get("leisure").map(|v| v == "park" || v == "nature_reserve" || v == "garden").unwrap_or(false) ||
tags.get("natural").map(|v| v == "wood" || v == "scrub" || v == "heath").unwrap_or(false);
if is_water || is_landuse {
// Collect outer way members
let mut outer_ways: Vec<i64> = Vec::new();
for member in rel.members() {
if member.role().unwrap_or("") == "outer" {
if let osmpbf::RelMemberType::Way = member.member_type {
outer_ways.push(member.member_id);
}
}
}
if !outer_ways.is_empty() {
// Assemble ALL rings from the outer ways (rivers have multiple rings!)
let rings = assemble_rings(&outer_ways, &way_store);
for ring_node_ids in rings {
// Resolve node coordinates
let mut points: Vec<(f64, f64)> = Vec::new();
for node_id in &ring_node_ids {
if let Some((lat, lon)) = node_store.get(*node_id) {
points.push((lat, lon));
}
}
if points.len() >= 4 {
let id = rel.id();
let (first_lat, first_lon) = points[0];
for &zoom in &ZOOM_LEVELS {
if !should_include(&tags, zoom) { continue; }
// No simplification for multipolygons
let final_points = triangulate_polygon(&points);
if final_points.len() < 3 { continue; }
let (x, y) = lat_lon_to_tile(first_lat, first_lon, zoom);
let zoom_i32 = zoom as i32;
// Create polygon blob
let mut polygon_blob = Vec::with_capacity(final_points.len() * 8);
for (lat, lon) in &final_points {
polygon_blob.extend_from_slice(&(*lat as f32).to_le_bytes());
polygon_blob.extend_from_slice(&(*lon as f32).to_le_bytes());
}
let table = if is_water { "water" } else { "landuse" };
let task = DbTask::Way { zoom: zoom_i32, table, id, tags: tags.clone(), points: polygon_blob.clone(), x, y };
let _ = tx.blocking_send(task);
}
}
}
}
}
}
}
_ => {}
}
if (node_count + way_count) % 100_000 == 0 {
println!("Processed {} nodes, {} ways...", node_count, way_count);
if (node_count + way_count + relation_count) % 100_000 == 0 {
println!("Processed {} nodes, {} ways, {} relations...", node_count, way_count, relation_count);
}
})?;
Ok((node_count, way_count))
Ok((node_count, way_count, relation_count))
});
let (node_count, way_count) = reader_handle.await??;
let (node_count, way_count, relation_count) = reader_handle.await??;
println!("Finished reading PBF. Nodes: {}, Ways: {}. Waiting for consumer...", node_count, way_count);
println!("Finished reading PBF. Nodes: {}, Ways: {}, Relations: {}. Waiting for consumer...", node_count, way_count, relation_count);
// Drop sender to signal consumer to finish
drop(tx);