diff --git a/backend/src/main.rs b/backend/src/main.rs index ffc8fd4..7e1bf3b 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -111,11 +111,7 @@ async fn get_tile_ways( Path((z, x, y)): Path<(i32, i32, i32)>, State(state): State>, ) -> impl IntoResponse { - let query = if z < 9 { - "SELECT id, tags, points FROM map_data.ways WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 10000" - } else { - "SELECT id, tags, points FROM map_data.ways WHERE zoom = ? AND tile_x = ? AND tile_y = ?" - }; + let query = "SELECT id, tags, points FROM map_data.ways WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 50000"; let rows = match state.scylla_session.query(query, (z, x, y)).await { Ok(res) => res.rows.unwrap_or_default(), Err(e) => return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, format!("Database error: {}", e)).into_response(), @@ -140,7 +136,7 @@ async fn get_tile_buildings( Path((z, x, y)): Path<(i32, i32, i32)>, State(state): State>, ) -> impl IntoResponse { - let query = "SELECT id, tags, points FROM map_data.buildings WHERE zoom = ? AND tile_x = ? AND tile_y = ?"; + let query = "SELECT id, tags, points FROM map_data.buildings WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 50000"; // Optimization: Don't load buildings for low zoom levels if z < 12 { @@ -176,7 +172,7 @@ async fn get_tile_landuse( return ([(header::CONTENT_TYPE, "application/octet-stream")], vec![]).into_response(); } - let query = "SELECT id, tags, points FROM map_data.landuse WHERE zoom = ? AND tile_x = ? AND tile_y = ?"; + let query = "SELECT id, tags, points FROM map_data.landuse WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 50000"; let rows = match state.scylla_session.query(query, (z, x, y)).await { Ok(res) => res.rows.unwrap_or_default(), Err(e) => return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, format!("Database error: {}", e)).into_response(), @@ -201,11 +197,7 @@ async fn get_tile_water( Path((z, x, y)): Path<(i32, i32, i32)>, State(state): State>, ) -> impl IntoResponse { - let query = if z < 9 { - "SELECT id, tags, points FROM map_data.water WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 10000" - } else { - "SELECT id, tags, points FROM map_data.water WHERE zoom = ? AND tile_x = ? AND tile_y = ?" - }; + let query = "SELECT id, tags, points FROM map_data.water WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 50000"; let rows = match state.scylla_session.query(query, (z, x, y)).await { Ok(res) => res.rows.unwrap_or_default(), Err(e) => return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, format!("Database error: {}", e)).into_response(), @@ -230,11 +222,7 @@ async fn get_tile_railways( Path((z, x, y)): Path<(i32, i32, i32)>, State(state): State>, ) -> impl IntoResponse { - let query = if z < 9 { - "SELECT id, tags, points FROM map_data.railways WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 10000" - } else { - "SELECT id, tags, points FROM map_data.railways WHERE zoom = ? AND tile_x = ? AND tile_y = ?" - }; + let query = "SELECT id, tags, points FROM map_data.railways WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 50000"; let rows = match state.scylla_session.query(query, (z, x, y)).await { Ok(res) => res.rows.unwrap_or_default(), Err(e) => return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, format!("Database error: {}", e)).into_response(), @@ -303,11 +291,7 @@ async fn get_tile_all( } // Ways - let query_ways = if z < 9 { - "SELECT id, tags, points FROM map_data.ways WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 10000" - } else { - "SELECT id, tags, points FROM map_data.ways WHERE zoom = ? AND tile_x = ? AND tile_y = ?" - }; + let query_ways = "SELECT id, tags, points FROM map_data.ways WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 50000"; let rows_ways = state.scylla_session.query(query_ways, (z, x, y)).await.ok().and_then(|r| r.rows).unwrap_or_default(); let mut ways = Vec::new(); for row in rows_ways { @@ -319,7 +303,7 @@ async fn get_tile_all( // Buildings let mut buildings = Vec::new(); if z >= 13 { - let query_buildings = "SELECT id, tags, points FROM map_data.buildings WHERE zoom = ? AND tile_x = ? AND tile_y = ?"; + let query_buildings = "SELECT id, tags, points FROM map_data.buildings WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 50000"; let rows_buildings = state.scylla_session.query(query_buildings, (z, x, y)).await.ok().and_then(|r| r.rows).unwrap_or_default(); for row in rows_buildings { if let Ok((id, tags, points)) = row.into_typed::<(i64, std::collections::HashMap, Vec)>() { @@ -331,7 +315,7 @@ async fn get_tile_all( // Landuse let mut landuse = Vec::new(); if z >= 4 { - let query_landuse = "SELECT id, tags, points FROM map_data.landuse WHERE zoom = ? AND tile_x = ? AND tile_y = ?"; + let query_landuse = "SELECT id, tags, points FROM map_data.landuse WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 50000"; let rows_landuse = state.scylla_session.query(query_landuse, (z, x, y)).await.ok().and_then(|r| r.rows).unwrap_or_default(); for row in rows_landuse { if let Ok((id, tags, points)) = row.into_typed::<(i64, std::collections::HashMap, Vec)>() { @@ -341,11 +325,7 @@ async fn get_tile_all( } // Water - let query_water = if z < 9 { - "SELECT id, tags, points FROM map_data.water WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 10000" - } else { - "SELECT id, tags, points FROM map_data.water WHERE zoom = ? AND tile_x = ? AND tile_y = ?" - }; + let query_water = "SELECT id, tags, points FROM map_data.water WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 50000"; let rows_water = state.scylla_session.query(query_water, (z, x, y)).await.ok().and_then(|r| r.rows).unwrap_or_default(); let mut water = Vec::new(); for row in rows_water { @@ -355,11 +335,7 @@ async fn get_tile_all( } // Railways - let query_railways = if z < 9 { - "SELECT id, tags, points FROM map_data.railways WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 10000" - } else { - "SELECT id, tags, points FROM map_data.railways WHERE zoom = ? AND tile_x = ? AND tile_y = ?" - }; + let query_railways = "SELECT id, tags, points FROM map_data.railways WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 50000"; let rows_railways = state.scylla_session.query(query_railways, (z, x, y)).await.ok().and_then(|r| r.rows).unwrap_or_default(); let mut railways = Vec::new(); for row in rows_railways { diff --git a/frontend/src/lib.rs b/frontend/src/lib.rs index 3252fee..86accc2 100644 --- a/frontend/src/lib.rs +++ b/frontend/src/lib.rs @@ -33,6 +33,8 @@ use tiles::{fetch_cached, get_visible_tiles, get_parent_tile}; use labels::update_labels; use pipelines::{ Vertex, + ColoredVertex, + create_simple_pipeline, create_building_pipeline, create_water_pipeline, create_water_line_pipeline, @@ -420,7 +422,7 @@ pub async fn run() { let mut landuse_residential_vertex_data: Vec = Vec::new(); let mut landuse_sand_vertex_data: Vec = Vec::new(); let mut water_vertex_data: Vec = Vec::new(); - let mut railway_vertex_data: Vec = Vec::new(); + let mut railway_vertex_data: Vec = Vec::new(); let mut water_line_vertex_data: Vec = Vec::new(); // Process ways (roads) @@ -518,17 +520,23 @@ pub async fn run() { } } - // Process railways + // Process railways with colors if let Some(railway_list) = state_guard.railways.get(&tile) { for railway in railway_list { + // Parse color from tags (format: "#RRGGBB" or "#RGB") + let color = railway.tags.get("colour") + .or(railway.tags.get("color")) + .map(|c| parse_hex_color(c)) + .unwrap_or([0.0, 0.0, 0.0]); // Default: no color (shader uses grey) + // Parse all points first - let mut rail_points: Vec = Vec::new(); + let mut rail_points: Vec = Vec::new(); for chunk in railway.points.chunks(8) { if chunk.len() < 8 { break; } let lat = f32::from_le_bytes(chunk[0..4].try_into().unwrap_or([0u8; 4])); let lon = f32::from_le_bytes(chunk[4..8].try_into().unwrap_or([0u8; 4])); let (x, y) = project(lat as f64, lon as f64); - rail_points.push(Vertex { position: [x, y] }); + rail_points.push(ColoredVertex { position: [x, y], color }); } // For LineList: push pairs of vertices for each segment @@ -897,3 +905,25 @@ pub async fn run() { } }).unwrap(); } + +/// Parse a hex color string (e.g., "#FF0000" or "#F00") into RGB floats [0.0-1.0] +fn parse_hex_color(hex: &str) -> [f32; 3] { + let hex = hex.trim_start_matches('#'); + + if hex.len() == 6 { + // #RRGGBB format + let r = u8::from_str_radix(&hex[0..2], 16).unwrap_or(0) as f32 / 255.0; + let g = u8::from_str_radix(&hex[2..4], 16).unwrap_or(0) as f32 / 255.0; + let b = u8::from_str_radix(&hex[4..6], 16).unwrap_or(0) as f32 / 255.0; + [r, g, b] + } else if hex.len() == 3 { + // #RGB format (shorthand) + let r = u8::from_str_radix(&hex[0..1], 16).unwrap_or(0) as f32 / 15.0; + let g = u8::from_str_radix(&hex[1..2], 16).unwrap_or(0) as f32 / 15.0; + let b = u8::from_str_radix(&hex[2..3], 16).unwrap_or(0) as f32 / 15.0; + [r, g, b] + } else { + // Invalid format, return black (will trigger grey fallback in shader) + [0.0, 0.0, 0.0] + } +} diff --git a/frontend/src/pipelines/common.rs b/frontend/src/pipelines/common.rs index 193a6c6..4c8f68c 100644 --- a/frontend/src/pipelines/common.rs +++ b/frontend/src/pipelines/common.rs @@ -23,6 +23,35 @@ impl Vertex { } } +/// GPU vertex with 2D position and RGB color (for railways with line colors) +#[repr(C)] +#[derive(Copy, Clone, Debug, bytemuck::Pod, bytemuck::Zeroable)] +pub struct ColoredVertex { + pub position: [f32; 2], + pub color: [f32; 3], +} + +impl ColoredVertex { + pub fn desc() -> wgpu::VertexBufferLayout<'static> { + wgpu::VertexBufferLayout { + array_stride: std::mem::size_of::() as wgpu::BufferAddress, + step_mode: wgpu::VertexStepMode::Vertex, + attributes: &[ + wgpu::VertexAttribute { + offset: 0, + shader_location: 0, + format: wgpu::VertexFormat::Float32x2, + }, + wgpu::VertexAttribute { + offset: std::mem::size_of::<[f32; 2]>() as wgpu::BufferAddress, + shader_location: 1, + format: wgpu::VertexFormat::Float32x3, + } + ] + } + } +} + /// Create a simple render pipeline with standard configuration pub fn create_simple_pipeline( device: &wgpu::Device, diff --git a/frontend/src/pipelines/mod.rs b/frontend/src/pipelines/mod.rs index 594a138..2305059 100644 --- a/frontend/src/pipelines/mod.rs +++ b/frontend/src/pipelines/mod.rs @@ -7,7 +7,7 @@ pub mod roads; pub mod landuse; pub mod railway; -pub use common::{Vertex, create_simple_pipeline}; +pub use common::{Vertex, ColoredVertex, create_simple_pipeline}; pub use building::create_building_pipeline; pub use water::{create_water_pipeline, create_water_line_pipeline}; pub use roads::{ diff --git a/frontend/src/pipelines/railway.rs b/frontend/src/pipelines/railway.rs index cb9aa95..5e16c18 100644 --- a/frontend/src/pipelines/railway.rs +++ b/frontend/src/pipelines/railway.rs @@ -1,6 +1,6 @@ //! Railway render pipeline -use super::common::Vertex; +use super::common::ColoredVertex; pub fn create_railway_pipeline( device: &wgpu::Device, @@ -19,10 +19,12 @@ pub fn create_railway_pipeline( struct VertexInput { @location(0) position: vec2, + @location(1) color: vec3, }; struct VertexOutput { @builtin(position) clip_position: vec4, + @location(0) color: vec3, }; @vertex @@ -35,21 +37,25 @@ pub fn create_railway_pipeline( let x = world_pos.x * camera.params.x + camera.params.z; let y = world_pos.y * camera.params.y + camera.params.w; - - // Globe Effect: Spherize - // let r2 = x*x + y*y; - // let w = 1.0 + r2 * 0.5; out.clip_position = vec4(x, y, 0.0, 1.0); + out.color = model.color; return out; } @fragment fn fs_main(in: VertexOutput) -> @location(0) vec4 { - let is_dark = camera.theme.x; - // Light: #808080 (grey), Dark: #5a5a5a (darker grey) - let color = mix(vec3(0.5, 0.5, 0.5), vec3(0.35, 0.35, 0.35), is_dark); - return vec4(color, 1.0); + // Use vertex color if it has any value, otherwise use default grey + let has_color = in.color.r > 0.01 || in.color.g > 0.01 || in.color.b > 0.01; + + if (has_color) { + return vec4(in.color, 1.0); + } else { + // Fallback: Light: #808080 (grey), Dark: #5a5a5a (darker grey) + let is_dark = camera.theme.x; + let color = mix(vec3(0.5, 0.5, 0.5), vec3(0.35, 0.35, 0.35), is_dark); + return vec4(color, 1.0); + } } "#)), }); @@ -67,7 +73,7 @@ pub fn create_railway_pipeline( module: &shader, entry_point: "vs_main", buffers: &[ - Vertex::desc(), + ColoredVertex::desc(), ], }, fragment: Some(wgpu::FragmentState { diff --git a/importer/src/main.rs b/importer/src/main.rs index 80af10c..963f0c5 100644 --- a/importer/src/main.rs +++ b/importer/src/main.rs @@ -30,6 +30,42 @@ impl WayStore { } } +// Store railway ways for deferred insertion (after relation processing for colors) +struct RailwayWay { + id: i64, + tags: HashMap, + points: Vec, // Serialized line blob + first_lat: f64, + first_lon: f64, +} + +struct RailwayStore { + ways: HashMap, // way_id -> railway data + way_colors: HashMap, // way_id -> colour from route relation +} + +impl RailwayStore { + fn new() -> Self { + Self { + ways: HashMap::new(), + way_colors: HashMap::new(), + } + } + + fn insert_way(&mut self, id: i64, tags: HashMap, points: Vec, first_lat: f64, first_lon: f64) { + self.ways.insert(id, RailwayWay { id, tags, points, first_lat, first_lon }); + } + + fn set_color(&mut self, way_id: i64, color: String) { + // Only set if not already set (first route relation wins) + self.way_colors.entry(way_id).or_insert(color); + } + + fn get_color(&self, way_id: i64) -> Option<&String> { + self.way_colors.get(&way_id) + } +} + // Assemble ways into MULTIPLE rings (connect end-to-end) // Rivers like the Isar have multiple separate channels/rings fn assemble_rings(way_ids: &[i64], way_store: &WayStore) -> Vec> { @@ -475,9 +511,12 @@ async fn main() -> Result<()> { // Store way geometries for multipolygon assembly let mut way_store = WayStore::new(); + + // Store railway ways for deferred insertion (after relation processing for colors) + let mut railway_store = RailwayStore::new(); - // We process sequentially: Nodes first, then Ways. - // osmpbf yields nodes then ways. + // We process sequentially: Nodes first, then Ways, then Relations. + // osmpbf yields nodes then ways then relations. // We need to detect when we switch from nodes to ways to prepare the store. reader.for_each(|element| { @@ -678,8 +717,9 @@ async fn main() -> Result<()> { } if is_railway { - let task = DbTask::Way { zoom: zoom_i32, table: "railways", id, tags: tags.clone(), points: line_blob.clone(), x, y }; - let _ = tx.blocking_send(task); + // Store for deferred insertion - colors will be applied from relations + let (first_lat, first_lon) = simplified_points[0]; + railway_store.insert_way(id, tags.clone(), line_blob.clone(), first_lat, first_lon); } } } @@ -694,7 +734,34 @@ async fn main() -> Result<()> { relation_count += 1; let tags: HashMap = rel.tags().map(|(k, v)| (k.to_string(), v.to_string())).collect(); - // Only process multipolygon relations + // Process route relations for transit colors + if tags.get("type").map(|t| t == "route").unwrap_or(false) { + let route_type = tags.get("route").map(|s| s.as_str()); + let is_transit = match route_type { + Some("subway") | Some("tram") | Some("light_rail") => true, + Some("train") => { + // Only include S-Bahn and suburban trains + tags.get("network").map(|n| n.contains("S-Bahn")).unwrap_or(false) || + tags.get("service").map(|s| s == "suburban").unwrap_or(false) || + tags.get("ref").map(|r| r.starts_with("S")).unwrap_or(false) + }, + _ => false, + }; + + if is_transit { + // Extract colour tag + if let Some(colour) = tags.get("colour").or(tags.get("color")) { + // Map colour to all member ways + for member in rel.members() { + if let osmpbf::RelMemberType::Way = member.member_type { + railway_store.set_color(member.member_id, colour.clone()); + } + } + } + } + } + + // Process multipolygon relations (existing code) if tags.get("type").map(|t| t == "multipolygon").unwrap_or(false) { // Check if it's a water or landuse multipolygon // IMPORTANT: Rivers like the Isar are tagged waterway=river on the relation itself! @@ -770,6 +837,37 @@ async fn main() -> Result<()> { } })?; + // Deferred railway insertion - now with colors from route relations + println!("Inserting {} railway ways with colors...", railway_store.ways.len()); + for (way_id, railway) in &railway_store.ways { + let mut tags = railway.tags.clone(); + + // Apply color from route relation if available + if let Some(colour) = railway_store.get_color(*way_id) { + tags.insert("colour".to_string(), colour.clone()); + } + + // Insert for all applicable zoom levels + for &zoom in &ZOOM_LEVELS { + if !should_include(&tags, zoom) { continue; } + + let (x, y) = lat_lon_to_tile(railway.first_lat, railway.first_lon, zoom); + let zoom_i32 = zoom as i32; + + let task = DbTask::Way { + zoom: zoom_i32, + table: "railways", + id: railway.id, + tags: tags.clone(), + points: railway.points.clone(), + x, + y + }; + let _ = tx.blocking_send(task); + } + } + println!("Railway insertion complete."); + Ok((node_count, way_count, relation_count)) }); @@ -785,7 +883,23 @@ async fn main() -> Result<()> { // Clean up cache let _ = std::fs::remove_file(cache_path); - println!("Done!"); + + // Run major compaction to clean up tombstones from TRUNCATE + println!("Running major compaction to clean up tombstones..."); + let tables = ["nodes", "ways", "buildings", "water", "landuse", "railways"]; + for table in &tables { + println!("Compacting map_data.{}...", table); + let query = format!("ALTER TABLE map_data.{} WITH gc_grace_seconds = 0", table); + let _ = session.query(query, &[]).await; + } + + // Force a flush to ensure all data is on disk before compaction + // Note: In ScyllaDB, compaction happens automatically, but we set gc_grace_seconds=0 + // to allow immediate tombstone cleanup. For manual compaction, use nodetool externally. + println!("Compaction settings updated. Tombstones will be cleaned during next compaction cycle."); + println!("For immediate compaction, run: docker exec scylla nodetool compact map_data"); + + println!("Import complete!"); Ok(()) }