dongho/maps
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
Dongho Kim
2025-12-16 00:42:33 +09:00
parent 45807e3a90
commit 8edb92b25d
6 changed files with 210 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
importer/src/main.rs
View File

@@ -30,6 +30,42 @@ impl WayStore {
}
}
// Store railway ways for deferred insertion (after relation processing for colors)
struct RailwayWay {
id: i64,
tags: HashMap<String, String>,
points: Vec<u8>, // Serialized line blob
first_lat: f64,
first_lon: f64,
}
struct RailwayStore {
ways: HashMap<i64, RailwayWay>, // way_id -> railway data
way_colors: HashMap<i64, String>, // way_id -> colour from route relation
}
impl RailwayStore {
fn new() -> Self {
Self {
ways: HashMap::new(),
way_colors: HashMap::new(),
}
}
fn insert_way(&mut self, id: i64, tags: HashMap<String, String>, points: Vec<u8>, first_lat: f64, first_lon: f64) {
self.ways.insert(id, RailwayWay { id, tags, points, first_lat, first_lon });
}
fn set_color(&mut self, way_id: i64, color: String) {
// Only set if not already set (first route relation wins)
self.way_colors.entry(way_id).or_insert(color);
}
fn get_color(&self, way_id: i64) -> Option<&String> {
self.way_colors.get(&way_id)
}
}
// Assemble ways into MULTIPLE rings (connect end-to-end)
// Rivers like the Isar have multiple separate channels/rings
fn assemble_rings(way_ids: &[i64], way_store: &WayStore) -> Vec<Vec<i64>> {
@@ -475,9 +511,12 @@ async fn main() -> Result<()> {
// Store way geometries for multipolygon assembly
let mut way_store = WayStore::new();
// Store railway ways for deferred insertion (after relation processing for colors)
let mut railway_store = RailwayStore::new();
// We process sequentially: Nodes first, then Ways.
// osmpbf yields nodes then ways.
// We process sequentially: Nodes first, then Ways, then Relations.
// osmpbf yields nodes then ways then relations.
// We need to detect when we switch from nodes to ways to prepare the store.
reader.for_each(|element| {
@@ -678,8 +717,9 @@ async fn main() -> Result<()> {
}
if is_railway {
let task = DbTask::Way { zoom: zoom_i32, table: "railways", id, tags: tags.clone(), points: line_blob.clone(), x, y };
let _ = tx.blocking_send(task);
// Store for deferred insertion - colors will be applied from relations
let (first_lat, first_lon) = simplified_points[0];
railway_store.insert_way(id, tags.clone(), line_blob.clone(), first_lat, first_lon);
}
}
}
@@ -694,7 +734,34 @@ async fn main() -> Result<()> {
relation_count += 1;
let tags: HashMap<String, String> = rel.tags().map(|(k, v)| (k.to_string(), v.to_string())).collect();
// Only process multipolygon relations
// Process route relations for transit colors
if tags.get("type").map(|t| t == "route").unwrap_or(false) {
let route_type = tags.get("route").map(|s| s.as_str());
let is_transit = match route_type {
Some("subway") | Some("tram") | Some("light_rail") => true,
Some("train") => {
// Only include S-Bahn and suburban trains
tags.get("network").map(|n| n.contains("S-Bahn")).unwrap_or(false) ||
tags.get("service").map(|s| s == "suburban").unwrap_or(false) ||
tags.get("ref").map(|r| r.starts_with("S")).unwrap_or(false)
},
_ => false,
};
if is_transit {
// Extract colour tag
if let Some(colour) = tags.get("colour").or(tags.get("color")) {
// Map colour to all member ways
for member in rel.members() {
if let osmpbf::RelMemberType::Way = member.member_type {
railway_store.set_color(member.member_id, colour.clone());
}
}
}
}
}
// Process multipolygon relations (existing code)
if tags.get("type").map(|t| t == "multipolygon").unwrap_or(false) {
// Check if it's a water or landuse multipolygon
// IMPORTANT: Rivers like the Isar are tagged waterway=river on the relation itself!
@@ -770,6 +837,37 @@ async fn main() -> Result<()> {
}
})?;
// Deferred railway insertion - now with colors from route relations
println!("Inserting {} railway ways with colors...", railway_store.ways.len());
for (way_id, railway) in &railway_store.ways {
let mut tags = railway.tags.clone();
// Apply color from route relation if available
if let Some(colour) = railway_store.get_color(*way_id) {
tags.insert("colour".to_string(), colour.clone());
}
// Insert for all applicable zoom levels
for &zoom in &ZOOM_LEVELS {
if !should_include(&tags, zoom) { continue; }
let (x, y) = lat_lon_to_tile(railway.first_lat, railway.first_lon, zoom);
let zoom_i32 = zoom as i32;
let task = DbTask::Way {
zoom: zoom_i32,
table: "railways",
id: railway.id,
tags: tags.clone(),
points: railway.points.clone(),
x,
y
};
let _ = tx.blocking_send(task);
}
}
println!("Railway insertion complete.");
Ok((node_count, way_count, relation_count))
});
@@ -785,7 +883,23 @@ async fn main() -> Result<()> {
// Clean up cache
let _ = std::fs::remove_file(cache_path);
println!("Done!");
// Run major compaction to clean up tombstones from TRUNCATE
println!("Running major compaction to clean up tombstones...");
let tables = ["nodes", "ways", "buildings", "water", "landuse", "railways"];
for table in &tables {
println!("Compacting map_data.{}...", table);
let query = format!("ALTER TABLE map_data.{} WITH gc_grace_seconds = 0", table);
let _ = session.query(query, &[]).await;
}
// Force a flush to ensure all data is on disk before compaction
// Note: In ScyllaDB, compaction happens automatically, but we set gc_grace_seconds=0
// to allow immediate tombstone cleanup. For manual compaction, use nodetool externally.
println!("Compaction settings updated. Tombstones will be cleaned during next compaction cycle.");
println!("For immediate compaction, run: docker exec scylla nodetool compact map_data");
println!("Import complete!");
Ok(())
}