update

backend/src/main.rs

@@ -111,11 +111,7 @@ async fn get_tile_ways(
     Path((z, x, y)): Path<(i32, i32, i32)>,
     State(state): State<Arc<AppState>>,
 ) -> impl IntoResponse {
-    let query = if z < 9 {
+    let query = "SELECT id, tags, points FROM map_data.ways WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 50000";
-        "SELECT id, tags, points FROM map_data.ways WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 10000"
-    } else {
-        "SELECT id, tags, points FROM map_data.ways WHERE zoom = ? AND tile_x = ? AND tile_y = ?"
-    };
     let rows = match state.scylla_session.query(query, (z, x, y)).await {
         Ok(res) => res.rows.unwrap_or_default(),
         Err(e) => return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, format!("Database error: {}", e)).into_response(),
@@ -140,7 +136,7 @@ async fn get_tile_buildings(
     Path((z, x, y)): Path<(i32, i32, i32)>,
     State(state): State<Arc<AppState>>,
 ) -> impl IntoResponse {
-    let query = "SELECT id, tags, points FROM map_data.buildings WHERE zoom = ? AND tile_x = ? AND tile_y = ?";
+    let query = "SELECT id, tags, points FROM map_data.buildings WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 50000";
     
     // Optimization: Don't load buildings for low zoom levels
     if z < 12 {
@@ -176,7 +172,7 @@ async fn get_tile_landuse(
         return ([(header::CONTENT_TYPE, "application/octet-stream")], vec![]).into_response();
     }
 
-    let query = "SELECT id, tags, points FROM map_data.landuse WHERE zoom = ? AND tile_x = ? AND tile_y = ?";
+    let query = "SELECT id, tags, points FROM map_data.landuse WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 50000";
     let rows = match state.scylla_session.query(query, (z, x, y)).await {
         Ok(res) => res.rows.unwrap_or_default(),
         Err(e) => return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, format!("Database error: {}", e)).into_response(),
@@ -201,11 +197,7 @@ async fn get_tile_water(
     Path((z, x, y)): Path<(i32, i32, i32)>,
     State(state): State<Arc<AppState>>,
 ) -> impl IntoResponse {
-    let query = if z < 9 {
+    let query = "SELECT id, tags, points FROM map_data.water WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 50000";
-        "SELECT id, tags, points FROM map_data.water WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 10000"
-    } else {
-        "SELECT id, tags, points FROM map_data.water WHERE zoom = ? AND tile_x = ? AND tile_y = ?"
-    };
     let rows = match state.scylla_session.query(query, (z, x, y)).await {
         Ok(res) => res.rows.unwrap_or_default(),
         Err(e) => return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, format!("Database error: {}", e)).into_response(),
@@ -230,11 +222,7 @@ async fn get_tile_railways(
     Path((z, x, y)): Path<(i32, i32, i32)>,
     State(state): State<Arc<AppState>>,
 ) -> impl IntoResponse {
-    let query = if z < 9 {
+    let query = "SELECT id, tags, points FROM map_data.railways WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 50000";
-        "SELECT id, tags, points FROM map_data.railways WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 10000"
-    } else {
-        "SELECT id, tags, points FROM map_data.railways WHERE zoom = ? AND tile_x = ? AND tile_y = ?"
-    };
     let rows = match state.scylla_session.query(query, (z, x, y)).await {
         Ok(res) => res.rows.unwrap_or_default(),
         Err(e) => return (axum::http::StatusCode::INTERNAL_SERVER_ERROR, format!("Database error: {}", e)).into_response(),
@@ -303,11 +291,7 @@ async fn get_tile_all(
     }
 
     // Ways
-    let query_ways = if z < 9 {
+    let query_ways = "SELECT id, tags, points FROM map_data.ways WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 50000";
-        "SELECT id, tags, points FROM map_data.ways WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 10000"
-    } else {
-        "SELECT id, tags, points FROM map_data.ways WHERE zoom = ? AND tile_x = ? AND tile_y = ?"
-    };
     let rows_ways = state.scylla_session.query(query_ways, (z, x, y)).await.ok().and_then(|r| r.rows).unwrap_or_default();
     let mut ways = Vec::new();
     for row in rows_ways {
@@ -319,7 +303,7 @@ async fn get_tile_all(
     // Buildings
     let mut buildings = Vec::new();
     if z >= 13 {
-        let query_buildings = "SELECT id, tags, points FROM map_data.buildings WHERE zoom = ? AND tile_x = ? AND tile_y = ?";
+        let query_buildings = "SELECT id, tags, points FROM map_data.buildings WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 50000";
         let rows_buildings = state.scylla_session.query(query_buildings, (z, x, y)).await.ok().and_then(|r| r.rows).unwrap_or_default();
         for row in rows_buildings {
             if let Ok((id, tags, points)) = row.into_typed::<(i64, std::collections::HashMap<String, String>, Vec<u8>)>() {
@@ -331,7 +315,7 @@ async fn get_tile_all(
     // Landuse
     let mut landuse = Vec::new();
     if z >= 4 {
-        let query_landuse = "SELECT id, tags, points FROM map_data.landuse WHERE zoom = ? AND tile_x = ? AND tile_y = ?";
+        let query_landuse = "SELECT id, tags, points FROM map_data.landuse WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 50000";
         let rows_landuse = state.scylla_session.query(query_landuse, (z, x, y)).await.ok().and_then(|r| r.rows).unwrap_or_default();
         for row in rows_landuse {
             if let Ok((id, tags, points)) = row.into_typed::<(i64, std::collections::HashMap<String, String>, Vec<u8>)>() {
@@ -341,11 +325,7 @@ async fn get_tile_all(
     }
 
     // Water
-    let query_water = if z < 9 {
+    let query_water = "SELECT id, tags, points FROM map_data.water WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 50000";
-        "SELECT id, tags, points FROM map_data.water WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 10000"
-    } else {
-        "SELECT id, tags, points FROM map_data.water WHERE zoom = ? AND tile_x = ? AND tile_y = ?"
-    };
     let rows_water = state.scylla_session.query(query_water, (z, x, y)).await.ok().and_then(|r| r.rows).unwrap_or_default();
     let mut water = Vec::new();
     for row in rows_water {
@@ -355,11 +335,7 @@ async fn get_tile_all(
     }
 
     // Railways
-    let query_railways = if z < 9 {
+    let query_railways = "SELECT id, tags, points FROM map_data.railways WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 50000";
-        "SELECT id, tags, points FROM map_data.railways WHERE zoom = ? AND tile_x = ? AND tile_y = ? LIMIT 10000"
-    } else {
-        "SELECT id, tags, points FROM map_data.railways WHERE zoom = ? AND tile_x = ? AND tile_y = ?"
-    };
     let rows_railways = state.scylla_session.query(query_railways, (z, x, y)).await.ok().and_then(|r| r.rows).unwrap_or_default();
     let mut railways = Vec::new();
     for row in rows_railways {

frontend/src/lib.rs

@@ -33,6 +33,8 @@ use tiles::{fetch_cached, get_visible_tiles, get_parent_tile};
 use labels::update_labels;
 use pipelines::{
     Vertex,
+    ColoredVertex,
+    create_simple_pipeline,
     create_building_pipeline,
     create_water_pipeline,
     create_water_line_pipeline,
@@ -420,7 +422,7 @@ pub async fn run() {
                             let mut landuse_residential_vertex_data: Vec<Vertex> = Vec::new();
                             let mut landuse_sand_vertex_data: Vec<Vertex> = Vec::new();
                             let mut water_vertex_data: Vec<Vertex> = Vec::new();
-                            let mut railway_vertex_data: Vec<Vertex> = Vec::new();
+                            let mut railway_vertex_data: Vec<ColoredVertex> = Vec::new();
                             let mut water_line_vertex_data: Vec<Vertex> = Vec::new();
                             
                             // Process ways (roads)
@@ -518,17 +520,23 @@ pub async fn run() {
                                 }
                             }
                             
-                            // Process railways
+                            // Process railways with colors
                             if let Some(railway_list) = state_guard.railways.get(&tile) {
                                 for railway in railway_list {
+                                    // Parse color from tags (format: "#RRGGBB" or "#RGB")
+                                    let color = railway.tags.get("colour")
+                                        .or(railway.tags.get("color"))
+                                        .map(|c| parse_hex_color(c))
+                                        .unwrap_or([0.0, 0.0, 0.0]); // Default: no color (shader uses grey)
+                                    
                                     // Parse all points first
-                                    let mut rail_points: Vec<Vertex> = Vec::new();
+                                    let mut rail_points: Vec<ColoredVertex> = Vec::new();
                                     for chunk in railway.points.chunks(8) {
                                         if chunk.len() < 8 { break; }
                                         let lat = f32::from_le_bytes(chunk[0..4].try_into().unwrap_or([0u8; 4]));
                                         let lon = f32::from_le_bytes(chunk[4..8].try_into().unwrap_or([0u8; 4]));
                                         let (x, y) = project(lat as f64, lon as f64);
-                                        rail_points.push(Vertex { position: [x, y] });
+                                        rail_points.push(ColoredVertex { position: [x, y], color });
                                     }
                                     
                                     // For LineList: push pairs of vertices for each segment
@@ -897,3 +905,25 @@ pub async fn run() {
         }
     }).unwrap();
 }
+
+/// Parse a hex color string (e.g., "#FF0000" or "#F00") into RGB floats [0.0-1.0]
+fn parse_hex_color(hex: &str) -> [f32; 3] {
+    let hex = hex.trim_start_matches('#');
+    
+    if hex.len() == 6 {
+        // #RRGGBB format
+        let r = u8::from_str_radix(&hex[0..2], 16).unwrap_or(0) as f32 / 255.0;
+        let g = u8::from_str_radix(&hex[2..4], 16).unwrap_or(0) as f32 / 255.0;
+        let b = u8::from_str_radix(&hex[4..6], 16).unwrap_or(0) as f32 / 255.0;
+        [r, g, b]
+    } else if hex.len() == 3 {
+        // #RGB format (shorthand)
+        let r = u8::from_str_radix(&hex[0..1], 16).unwrap_or(0) as f32 / 15.0;
+        let g = u8::from_str_radix(&hex[1..2], 16).unwrap_or(0) as f32 / 15.0;
+        let b = u8::from_str_radix(&hex[2..3], 16).unwrap_or(0) as f32 / 15.0;
+        [r, g, b]
+    } else {
+        // Invalid format, return black (will trigger grey fallback in shader)
+        [0.0, 0.0, 0.0]
+    }
+}

frontend/src/pipelines/common.rs

@@ -23,6 +23,35 @@ impl Vertex {
     }
 }
 
+/// GPU vertex with 2D position and RGB color (for railways with line colors)
+#[repr(C)]
+#[derive(Copy, Clone, Debug, bytemuck::Pod, bytemuck::Zeroable)]
+pub struct ColoredVertex {
+    pub position: [f32; 2],
+    pub color: [f32; 3],
+}
+
+impl ColoredVertex {
+    pub fn desc() -> wgpu::VertexBufferLayout<'static> {
+        wgpu::VertexBufferLayout {
+            array_stride: std::mem::size_of::<ColoredVertex>() as wgpu::BufferAddress,
+            step_mode: wgpu::VertexStepMode::Vertex,
+            attributes: &[
+                wgpu::VertexAttribute {
+                    offset: 0,
+                    shader_location: 0,
+                    format: wgpu::VertexFormat::Float32x2,
+                },
+                wgpu::VertexAttribute {
+                    offset: std::mem::size_of::<[f32; 2]>() as wgpu::BufferAddress,
+                    shader_location: 1,
+                    format: wgpu::VertexFormat::Float32x3,
+                }
+            ]
+        }
+    }
+}
+
 /// Create a simple render pipeline with standard configuration
 pub fn create_simple_pipeline(
     device: &wgpu::Device, 

frontend/src/pipelines/mod.rs

@@ -7,7 +7,7 @@ pub mod roads;
 pub mod landuse;
 pub mod railway;
 
-pub use common::{Vertex, create_simple_pipeline};
+pub use common::{Vertex, ColoredVertex, create_simple_pipeline};
 pub use building::create_building_pipeline;
 pub use water::{create_water_pipeline, create_water_line_pipeline};
 pub use roads::{

frontend/src/pipelines/railway.rs

@@ -1,6 +1,6 @@
 //! Railway render pipeline
 
-use super::common::Vertex;
+use super::common::ColoredVertex;
 
 pub fn create_railway_pipeline(
     device: &wgpu::Device, 
@@ -19,10 +19,12 @@ pub fn create_railway_pipeline(
 
             struct VertexInput {
                 @location(0) position: vec2<f32>,
+                @location(1) color: vec3<f32>,
             };
 
             struct VertexOutput {
                 @builtin(position) clip_position: vec4<f32>,
+                @location(0) color: vec3<f32>,
             };
 
             @vertex
@@ -36,20 +38,24 @@ pub fn create_railway_pipeline(
                 let x = world_pos.x * camera.params.x + camera.params.z;
                 let y = world_pos.y * camera.params.y + camera.params.w;
                 
-                // Globe Effect: Spherize
-                // let r2 = x*x + y*y;
-                // let w = 1.0 + r2 * 0.5;
-                
                 out.clip_position = vec4<f32>(x, y, 0.0, 1.0);
+                out.color = model.color;
                 return out;
             }
 
             @fragment
             fn fs_main(in: VertexOutput) -> @location(0) vec4<f32> {
-                let is_dark = camera.theme.x;
+                // Use vertex color if it has any value, otherwise use default grey
-                // Light: #808080 (grey), Dark: #5a5a5a (darker grey)
+                let has_color = in.color.r > 0.01 || in.color.g > 0.01 || in.color.b > 0.01;
-                let color = mix(vec3<f32>(0.5, 0.5, 0.5), vec3<f32>(0.35, 0.35, 0.35), is_dark);
+                
-                return vec4<f32>(color, 1.0);
+                if (has_color) {
+                    return vec4<f32>(in.color, 1.0);
+                } else {
+                    // Fallback: Light: #808080 (grey), Dark: #5a5a5a (darker grey)
+                    let is_dark = camera.theme.x;
+                    let color = mix(vec3<f32>(0.5, 0.5, 0.5), vec3<f32>(0.35, 0.35, 0.35), is_dark);
+                    return vec4<f32>(color, 1.0);
+                }
             }
         "#)),
     });
@@ -67,7 +73,7 @@ pub fn create_railway_pipeline(
             module: &shader,
             entry_point: "vs_main",
             buffers: &[
-                Vertex::desc(),
+                ColoredVertex::desc(),
             ],
         },
         fragment: Some(wgpu::FragmentState {

importer/src/main.rs

@@ -30,6 +30,42 @@ impl WayStore {
     }
 }
 
+// Store railway ways for deferred insertion (after relation processing for colors)
+struct RailwayWay {
+    id: i64,
+    tags: HashMap<String, String>,
+    points: Vec<u8>,  // Serialized line blob
+    first_lat: f64,
+    first_lon: f64,
+}
+
+struct RailwayStore {
+    ways: HashMap<i64, RailwayWay>,  // way_id -> railway data
+    way_colors: HashMap<i64, String>, // way_id -> colour from route relation
+}
+
+impl RailwayStore {
+    fn new() -> Self {
+        Self { 
+            ways: HashMap::new(),
+            way_colors: HashMap::new(),
+        }
+    }
+    
+    fn insert_way(&mut self, id: i64, tags: HashMap<String, String>, points: Vec<u8>, first_lat: f64, first_lon: f64) {
+        self.ways.insert(id, RailwayWay { id, tags, points, first_lat, first_lon });
+    }
+    
+    fn set_color(&mut self, way_id: i64, color: String) {
+        // Only set if not already set (first route relation wins)
+        self.way_colors.entry(way_id).or_insert(color);
+    }
+    
+    fn get_color(&self, way_id: i64) -> Option<&String> {
+        self.way_colors.get(&way_id)
+    }
+}
+
 // Assemble ways into MULTIPLE rings (connect end-to-end)
 // Rivers like the Isar have multiple separate channels/rings
 fn assemble_rings(way_ids: &[i64], way_store: &WayStore) -> Vec<Vec<i64>> {
@@ -476,8 +512,11 @@ async fn main() -> Result<()> {
         // Store way geometries for multipolygon assembly
         let mut way_store = WayStore::new();
         
-        // We process sequentially: Nodes first, then Ways.
+        // Store railway ways for deferred insertion (after relation processing for colors)
-        // osmpbf yields nodes then ways.
+        let mut railway_store = RailwayStore::new();
+
+        // We process sequentially: Nodes first, then Ways, then Relations.
+        // osmpbf yields nodes then ways then relations.
         // We need to detect when we switch from nodes to ways to prepare the store.
         
         reader.for_each(|element| {
@@ -678,8 +717,9 @@ async fn main() -> Result<()> {
                                 }
 
                                 if is_railway {
-                                    let task = DbTask::Way { zoom: zoom_i32, table: "railways", id, tags: tags.clone(), points: line_blob.clone(), x, y };
+                                    // Store for deferred insertion - colors will be applied from relations
-                                    let _ = tx.blocking_send(task);
+                                    let (first_lat, first_lon) = simplified_points[0];
+                                    railway_store.insert_way(id, tags.clone(), line_blob.clone(), first_lat, first_lon);
                                 }
                             }
                         }
@@ -694,7 +734,34 @@ async fn main() -> Result<()> {
                     relation_count += 1;
                     let tags: HashMap<String, String> = rel.tags().map(|(k, v)| (k.to_string(), v.to_string())).collect();
                     
-                    // Only process multipolygon relations
+                    // Process route relations for transit colors
+                    if tags.get("type").map(|t| t == "route").unwrap_or(false) {
+                        let route_type = tags.get("route").map(|s| s.as_str());
+                        let is_transit = match route_type {
+                            Some("subway") | Some("tram") | Some("light_rail") => true,
+                            Some("train") => {
+                                // Only include S-Bahn and suburban trains
+                                tags.get("network").map(|n| n.contains("S-Bahn")).unwrap_or(false) ||
+                                tags.get("service").map(|s| s == "suburban").unwrap_or(false) ||
+                                tags.get("ref").map(|r| r.starts_with("S")).unwrap_or(false)
+                            },
+                            _ => false,
+                        };
+                        
+                        if is_transit {
+                            // Extract colour tag
+                            if let Some(colour) = tags.get("colour").or(tags.get("color")) {
+                                // Map colour to all member ways
+                                for member in rel.members() {
+                                    if let osmpbf::RelMemberType::Way = member.member_type {
+                                        railway_store.set_color(member.member_id, colour.clone());
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    
+                    // Process multipolygon relations (existing code)
                     if tags.get("type").map(|t| t == "multipolygon").unwrap_or(false) {
                         // Check if it's a water or landuse multipolygon
                         // IMPORTANT: Rivers like the Isar are tagged waterway=river on the relation itself!
@@ -770,6 +837,37 @@ async fn main() -> Result<()> {
             }
         })?;
         
+        // Deferred railway insertion - now with colors from route relations
+        println!("Inserting {} railway ways with colors...", railway_store.ways.len());
+        for (way_id, railway) in &railway_store.ways {
+            let mut tags = railway.tags.clone();
+            
+            // Apply color from route relation if available
+            if let Some(colour) = railway_store.get_color(*way_id) {
+                tags.insert("colour".to_string(), colour.clone());
+            }
+            
+            // Insert for all applicable zoom levels
+            for &zoom in &ZOOM_LEVELS {
+                if !should_include(&tags, zoom) { continue; }
+                
+                let (x, y) = lat_lon_to_tile(railway.first_lat, railway.first_lon, zoom);
+                let zoom_i32 = zoom as i32;
+                
+                let task = DbTask::Way { 
+                    zoom: zoom_i32, 
+                    table: "railways", 
+                    id: railway.id, 
+                    tags: tags.clone(), 
+                    points: railway.points.clone(), 
+                    x, 
+                    y 
+                };
+                let _ = tx.blocking_send(task);
+            }
+        }
+        println!("Railway insertion complete.");
+        
         Ok((node_count, way_count, relation_count))
     });
 
@@ -785,7 +883,23 @@ async fn main() -> Result<()> {
     
     // Clean up cache
     let _ = std::fs::remove_file(cache_path);
-    println!("Done!");
+    
+    // Run major compaction to clean up tombstones from TRUNCATE
+    println!("Running major compaction to clean up tombstones...");
+    let tables = ["nodes", "ways", "buildings", "water", "landuse", "railways"];
+    for table in &tables {
+        println!("Compacting map_data.{}...", table);
+        let query = format!("ALTER TABLE map_data.{} WITH gc_grace_seconds = 0", table);
+        let _ = session.query(query, &[]).await;
+    }
+    
+    // Force a flush to ensure all data is on disk before compaction
+    // Note: In ScyllaDB, compaction happens automatically, but we set gc_grace_seconds=0
+    // to allow immediate tombstone cleanup. For manual compaction, use nodetool externally.
+    println!("Compaction settings updated. Tombstones will be cleaned during next compaction cycle.");
+    println!("For immediate compaction, run: docker exec scylla nodetool compact map_data");
+    
+    println!("Import complete!");
 
     Ok(())
 }