dongho/maps
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
Dongho Kim
2025-11-25 18:18:03 +01:00
parent f5f5f10338
commit 410fc79056
6 changed files with 464 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@@ -1 +1,6 @@
/target
scylla_data/
*.wasm
pkg/
node_modules/
.DS_Store

30
backend/src/db.rs
View File

@@ -59,6 +59,36 @@ pub async fn initialize_schema(session: &Session) -> Result<(), Box<dyn std::err
)
.await?;
session
.query(
"CREATE TABLE IF NOT EXISTS map_data.landuse (
zoom int,
tile_x int,
tile_y int,
id bigint,
tags map<text, text>,
points blob,
PRIMARY KEY ((zoom, tile_x, tile_y), id)
)",
&[],
)
.await?;
session
.query(
"CREATE TABLE IF NOT EXISTS map_data.water (
zoom int,
tile_x int,
tile_y int,
id bigint,
tags map<text, text>,
points blob,
PRIMARY KEY ((zoom, tile_x, tile_y), id)
)",
&[],
)
.await?;
println!("Schema initialized.");
Ok(())
}

55
backend/src/main.rs
View File

@@ -22,6 +22,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
tracing_subscriber::fmt::init();
println!("Connecting to ScyllaDB...");
println!("Starting backend with landuse support...");
let uri = std::env::var("SCYLLA_URI").unwrap_or_else(|_| "127.0.0.1:9042".to_string());
let session = SessionBuilder::new()
@@ -45,6 +46,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.route("/api/tiles/:z/:x/:y", get(get_tile))
.route("/api/tiles/:z/:x/:y/ways", get(get_tile_ways))
.route("/api/tiles/:z/:x/:y/buildings", get(get_tile_buildings))
.route("/api/tiles/:z/:x/:y/landuse", get(get_tile_landuse))
.route("/api/tiles/:z/:x/:y/water", get(get_tile_water))
.nest_service("/", ServeDir::new("static"))
.layer(CorsLayer::permissive())
.with_state(state);
@@ -144,3 +147,55 @@ async fn get_tile_buildings(
Json(buildings)
}
async fn get_tile_landuse(
Path((z, x, y)): Path<(i32, i32, i32)>,
State(state): State<Arc<AppState>>,
) -> Json<Vec<MapWay>> {
let query = "SELECT id, tags, points FROM map_data.landuse WHERE zoom = ? AND tile_x = ? AND tile_y = ?";
let rows = state.scylla_session.query(query, (z, x, y)).await.unwrap().rows.unwrap_or_default();
let mut landuse = Vec::new();
for row in rows {
let (id, tags, points_blob) = row.into_typed::<(i64, std::collections::HashMap<String, String>, Vec<u8>)>().unwrap();
let mut points = Vec::new();
for chunk in points_blob.chunks(16) {
if chunk.len() == 16 {
let lat = f64::from_be_bytes(chunk[0..8].try_into().unwrap());
let lon = f64::from_be_bytes(chunk[8..16].try_into().unwrap());
points.push(vec![lat, lon]);
}
}
landuse.push(MapWay { id, tags, points });
}
Json(landuse)
}
async fn get_tile_water(
Path((z, x, y)): Path<(i32, i32, i32)>,
State(state): State<Arc<AppState>>,
) -> Json<Vec<MapWay>> {
let query = "SELECT id, tags, points FROM map_data.water WHERE zoom = ? AND tile_x = ? AND tile_y = ?";
let rows = state.scylla_session.query(query, (z, x, y)).await.unwrap().rows.unwrap_or_default();
let mut water = Vec::new();
for row in rows {
let (id, tags, points_blob) = row.into_typed::<(i64, std::collections::HashMap<String, String>, Vec<u8>)>().unwrap();
let mut points = Vec::new();
for chunk in points_blob.chunks(16) {
if chunk.len() == 16 {
let lat = f64::from_be_bytes(chunk[0..8].try_into().unwrap());
let lon = f64::from_be_bytes(chunk[8..16].try_into().unwrap());
points.push(vec![lat, lon]);
}
}
water.push(MapWay { id, tags, points });
}
Json(water)
}

2
docker-compose.yml
View File

@@ -8,7 +8,7 @@ services:
- "9042:9042"
- "9160:9160"
- "10000:10000"
command: --smp 1 --memory 750M --overprovisioned 1 --api-address 0.0.0.0 --max-memory-for-unlimited-query-soft-limit 10485760 --tombstone-warn-threshold 100000
command: --smp 1 --memory 2G --overprovisioned 1 --api-address 0.0.0.0 --max-memory-for-unlimited-query-soft-limit 1073741824 --tombstone-warn-threshold 10000000
volumes:
- scylla_data:/var/lib/scylla

335
frontend/src/lib.rs
View File

@@ -129,12 +129,18 @@ struct TileBuffers {
road_vertex_count: u32,
building_vertex_buffer: wgpu::Buffer,
building_index_count: u32,
landuse_vertex_buffer: wgpu::Buffer,
landuse_index_count: u32,
water_vertex_buffer: wgpu::Buffer,
water_index_count: u32,
}
struct AppState {
nodes: HashMap<(i32, i32, i32), Vec<MapNode>>,
ways: HashMap<(i32, i32, i32), Vec<MapWay>>,
buildings: HashMap<(i32, i32, i32), Vec<MapWay>>,
landuse: HashMap<(i32, i32, i32), Vec<MapWay>>,
water: HashMap<(i32, i32, i32), Vec<MapWay>>,
buffers: HashMap<(i32, i32, i32), std::sync::Arc<TileBuffers>>,
loaded_tiles: HashSet<(i32, i32, i32)>,
pending_tiles: HashSet<(i32, i32, i32)>,
@@ -240,7 +246,8 @@ fn get_visible_tiles(camera: &Camera) -> Vec<(i32, i32, i32)> {
#[wasm_bindgen(start)]
pub async fn run() {
console_log::init_with_level(log::Level::Warn).expect("Could not initialize logger");
std::panic::set_hook(Box::new(console_error_panic_hook::hook));
console_log::init_with_level(log::Level::Warn).expect("Couldn't initialize logger");
std::panic::set_hook(Box::new(console_error_panic_hook::hook));
let event_loop = EventLoop::new().unwrap();
@@ -331,6 +338,8 @@ pub async fn run() {
nodes: HashMap::new(),
ways: HashMap::new(),
buildings: HashMap::new(),
landuse: HashMap::new(),
water: HashMap::new(),
buffers: HashMap::new(),
loaded_tiles: HashSet::new(),
pending_tiles: HashSet::new(),
@@ -440,6 +449,8 @@ pub async fn run() {
let pipeline = create_pipeline(&device, &config.format, &camera_bind_group_layout);
let road_pipeline = create_road_pipeline(&device, &config.format, &camera_bind_group_layout);
let building_pipeline = create_building_pipeline(&device, &config.format, &camera_bind_group_layout);
let landuse_pipeline = create_landuse_pipeline(&device, &config.format, &camera_bind_group_layout);
let water_pipeline = create_water_pipeline(&device, &config.format, &camera_bind_group_layout);
let window_clone = window.clone();
@@ -547,6 +558,8 @@ pub async fn run() {
let mut point_instance_data = Vec::new();
let mut road_vertex_data = Vec::new();
let mut building_vertex_data = Vec::new();
let mut landuse_vertex_data = Vec::new();
let mut water_vertex_data = Vec::new();
// Process nodes
if let Some(nodes) = state_guard.nodes.get(tile) {
@@ -589,7 +602,10 @@ pub async fn run() {
}
// Earcut triangulation
let indices = earcut(&flat_points, &[], 2).unwrap();
let indices = match earcut(&flat_points, &[], 2) {
Ok(i) => i,
Err(_) => continue,
};
for i in indices {
let p = projected_points[i];
@@ -597,9 +613,61 @@ pub async fn run() {
}
}
}
// Process landuse
if let Some(landuse) = state_guard.landuse.get(tile) {
for area in landuse {
if area.points.len() < 3 { continue; }
let mut flat_points = Vec::new();
let mut projected_points = Vec::new();
for p in &area.points {
let (x, y) = project(p[0], p[1]);
flat_points.push(x as f64);
flat_points.push(y as f64);
projected_points.push([x, y]);
}
let indices = match earcut(&flat_points, &[], 2) {
Ok(i) => i,
Err(_) => continue,
};
for i in indices {
let p = projected_points[i];
landuse_vertex_data.push(Vertex { position: p });
}
}
}
// Process water
if let Some(water) = state_guard.water.get(tile) {
for area in water {
if area.points.len() < 3 { continue; }
let mut flat_points = Vec::new();
let mut projected_points = Vec::new();
for p in &area.points {
let (x, y) = project(p[0], p[1]);
flat_points.push(x as f64);
flat_points.push(y as f64);
projected_points.push([x, y]);
}
let indices = match earcut(&flat_points, &[], 2) {
Ok(i) => i,
Err(_) => continue,
};
for i in indices {
let p = projected_points[i];
water_vertex_data.push(Vertex { position: p });
}
}
}
// Only create buffers if we have data
if !point_instance_data.is_empty() || !road_vertex_data.is_empty() || !building_vertex_data.is_empty() {
if !point_instance_data.is_empty() || !road_vertex_data.is_empty() || !building_vertex_data.is_empty() || !landuse_vertex_data.is_empty() || !water_vertex_data.is_empty() {
let point_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some("Tile Instance Buffer"),
contents: bytemuck::cast_slice(&point_instance_data),
@@ -618,6 +686,19 @@ pub async fn run() {
usage: wgpu::BufferUsages::VERTEX,
});
let landuse_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some("Tile Landuse Buffer"),
contents: bytemuck::cast_slice(&landuse_vertex_data),
usage: wgpu::BufferUsages::VERTEX,
});
web_sys::console::log_1(&format!("Created landuse buffer with {} vertices", landuse_vertex_data.len()).into());
let water_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some("Tile Water Buffer"),
contents: bytemuck::cast_slice(&water_vertex_data),
usage: wgpu::BufferUsages::VERTEX,
});
state_guard.buffers.insert(*tile, std::sync::Arc::new(TileBuffers {
point_instance_buffer: point_buffer,
point_count: point_instance_data.len() as u32,
@@ -625,6 +706,10 @@ pub async fn run() {
road_vertex_count: road_vertex_data.len() as u32,
building_vertex_buffer: building_buffer,
building_index_count: building_vertex_data.len() as u32,
landuse_vertex_buffer: landuse_buffer,
landuse_index_count: landuse_vertex_data.len() as u32,
water_vertex_buffer: water_buffer,
water_index_count: water_vertex_data.len() as u32,
}));
}
}
@@ -670,6 +755,22 @@ pub async fn run() {
None
};
// Fetch landuse
let url_landuse = format!("http://localhost:3000/api/tiles/{}/{}/{}/landuse", z, x, y);
let landuse_data = if let Some(json) = fetch_cached(&url_landuse).await {
serde_json::from_str::<Vec<MapWay>>(&json).ok()
} else {
None
};
// Fetch water
let url_water = format!("http://localhost:3000/api/tiles/{}/{}/{}/water", z, x, y);
let water_data = if let Some(json) = fetch_cached(&url_water).await {
serde_json::from_str::<Vec<MapWay>>(&json).ok()
} else {
None
};
let mut guard = state_clone.lock().unwrap();
if let Some(nodes) = nodes_data {
@@ -684,6 +785,20 @@ pub async fn run() {
guard.buildings.insert((z, x, y), buildings);
}
if let Some(landuse) = landuse_data {
if !landuse.is_empty() {
web_sys::console::log_1(&format!("Fetched {} landuse items for tile {}/{}/{}", landuse.len(), z, x, y).into());
}
guard.landuse.insert((z, x, y), landuse);
}
if let Some(water) = water_data {
if !water.is_empty() {
web_sys::console::log_1(&format!("Fetched {} water items for tile {}/{}/{}", water.len(), z, x, y).into());
}
guard.water.insert((z, x, y), water);
}
guard.loaded_tiles.insert((z, x, y));
guard.pending_tiles.remove(&(z, x, y));
@@ -696,7 +811,13 @@ pub async fn run() {
camera_uniform = camera_uniform_data;
queue.write_buffer(&camera_buffer, 0, bytemuck::cast_slice(&[camera_uniform]));
let frame = surface.get_current_texture().unwrap();
let frame = match surface.get_current_texture() {
Ok(frame) => frame,
Err(e) => {
web_sys::console::log_1(&format!("Surface error: {:?}", e).into());
return;
}
};
let view = frame.texture.create_view(&wgpu::TextureViewDescriptor::default());
let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: None });
@@ -736,7 +857,23 @@ pub async fn run() {
// Draw each tile - order: Roads -> Buildings -> Points (back to front)
for buffers in &tiles_to_render {
// Draw Roads (bottom layer)
// Draw Water (bottom layer)
if buffers.water_index_count > 0 {
rpass.set_pipeline(&water_pipeline);
rpass.set_bind_group(0, &camera_bind_group, &[]);
rpass.set_vertex_buffer(0, buffers.water_vertex_buffer.slice(..));
rpass.draw(0..buffers.water_index_count, 0..1);
}
// Draw Landuse (second layer)
if buffers.landuse_index_count > 0 {
rpass.set_pipeline(&landuse_pipeline);
rpass.set_bind_group(0, &camera_bind_group, &[]);
rpass.set_vertex_buffer(0, buffers.landuse_vertex_buffer.slice(..));
rpass.draw(0..buffers.landuse_index_count, 0..1);
}
// Draw Roads (third layer)
if buffers.road_vertex_count > 0 {
rpass.set_pipeline(&road_pipeline);
rpass.set_bind_group(0, &camera_bind_group, &[]);
@@ -1045,3 +1182,191 @@ fn create_building_pipeline(
multiview: None,
})
}
fn create_landuse_pipeline(
device: &wgpu::Device,
format: &wgpu::TextureFormat,
bind_group_layout: &wgpu::BindGroupLayout
) -> wgpu::RenderPipeline {
let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: None,
source: wgpu::ShaderSource::Wgsl(std::borrow::Cow::Borrowed(r#"
struct CameraUniform {
params: vec4<f32>,
};
@group(0) @binding(0)
var<uniform> camera: CameraUniform;
struct VertexInput {
@location(0) position: vec2<f32>,
};
struct VertexOutput {
@builtin(position) clip_position: vec4<f32>,
};
@vertex
fn vs_main(
model: VertexInput,
) -> VertexOutput {
var out: VertexOutput;
let world_pos = model.position;
let x = world_pos.x * camera.params.x + camera.params.z;
let y = world_pos.y * camera.params.y + camera.params.w;
out.clip_position = vec4<f32>(x, y, 0.0, 1.0);
return out;
}
@fragment
fn fs_main(in: VertexOutput) -> @location(0) vec4<f32> {
return vec4<f32>(0.6, 0.8, 0.6, 1.0); // Light green for parks
}
"#)),
});
let render_pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("Landuse Pipeline Layout"),
bind_group_layouts: &[bind_group_layout],
push_constant_ranges: &[],
});
device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
label: Some("Landuse Pipeline"),
layout: Some(&render_pipeline_layout),
vertex: wgpu::VertexState {
module: &shader,
entry_point: "vs_main",
buffers: &[
wgpu::VertexBufferLayout {
array_stride: std::mem::size_of::<Vertex>() as wgpu::BufferAddress,
step_mode: wgpu::VertexStepMode::Vertex,
attributes: &[
wgpu::VertexAttribute {
offset: 0,
shader_location: 0,
format: wgpu::VertexFormat::Float32x2,
}
],
}
],
},
fragment: Some(wgpu::FragmentState {
module: &shader,
entry_point: "fs_main",
targets: &[Some(wgpu::ColorTargetState {
format: *format,
blend: Some(wgpu::BlendState::REPLACE),
write_mask: wgpu::ColorWrites::ALL,
})],
}),
primitive: wgpu::PrimitiveState {
topology: wgpu::PrimitiveTopology::TriangleList,
strip_index_format: None,
front_face: wgpu::FrontFace::Ccw,
cull_mode: None,
unclipped_depth: false,
polygon_mode: wgpu::PolygonMode::Fill,
conservative: false,
},
depth_stencil: None,
multisample: wgpu::MultisampleState::default(),
multiview: None,
})
}
fn create_water_pipeline(
device: &wgpu::Device,
format: &wgpu::TextureFormat,
bind_group_layout: &wgpu::BindGroupLayout
) -> wgpu::RenderPipeline {
let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: None,
source: wgpu::ShaderSource::Wgsl(std::borrow::Cow::Borrowed(r#"
struct CameraUniform {
params: vec4<f32>,
};
@group(0) @binding(0)
var<uniform> camera: CameraUniform;
struct VertexInput {
@location(0) position: vec2<f32>,
};
struct VertexOutput {
@builtin(position) clip_position: vec4<f32>,
};
@vertex
fn vs_main(
model: VertexInput,
) -> VertexOutput {
var out: VertexOutput;
let world_pos = model.position;
let x = world_pos.x * camera.params.x + camera.params.z;
let y = world_pos.y * camera.params.y + camera.params.w;
out.clip_position = vec4<f32>(x, y, 0.0, 1.0);
return out;
}
@fragment
fn fs_main(in: VertexOutput) -> @location(0) vec4<f32> {
return vec4<f32>(0.6, 0.7, 0.9, 1.0); // Light blue for water
}
"#)),
});
let render_pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("Water Pipeline Layout"),
bind_group_layouts: &[bind_group_layout],
push_constant_ranges: &[],
});
device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
label: Some("Water Pipeline"),
layout: Some(&render_pipeline_layout),
vertex: wgpu::VertexState {
module: &shader,
entry_point: "vs_main",
buffers: &[
wgpu::VertexBufferLayout {
array_stride: std::mem::size_of::<Vertex>() as wgpu::BufferAddress,
step_mode: wgpu::VertexStepMode::Vertex,
attributes: &[
wgpu::VertexAttribute {
offset: 0,
shader_location: 0,
format: wgpu::VertexFormat::Float32x2,
}
],
}
],
},
fragment: Some(wgpu::FragmentState {
module: &shader,
entry_point: "fs_main",
targets: &[Some(wgpu::ColorTargetState {
format: *format,
blend: Some(wgpu::BlendState::REPLACE),
write_mask: wgpu::ColorWrites::ALL,
})],
}),
primitive: wgpu::PrimitiveState {
topology: wgpu::PrimitiveTopology::TriangleList,
strip_index_format: None,
front_face: wgpu::FrontFace::Ccw,
cull_mode: None,
unclipped_depth: false,
polygon_mode: wgpu::PolygonMode::Fill,
conservative: false,
},
depth_stencil: None,
multisample: wgpu::MultisampleState::default(),
multiview: None,
})
}

48
importer/src/main.rs
View File

@@ -31,6 +31,9 @@ async fn main() -> Result<()> {
let mut way_count = 0;
let mut inserted_nodes = 0;
let mut inserted_ways = 0;
let mut inserted_buildings = 0;
let mut inserted_water = 0;
let mut inserted_landuse = 0;
// We process sequentially: Nodes first, then Ways.
reader.for_each(|element| {
@@ -83,11 +86,17 @@ async fn main() -> Result<()> {
way_count += 1;
let tags: HashMap<String, String> = way.tags().map(|(k, v)| (k.to_string(), v.to_string())).collect();
// Filter for highways/roads OR buildings
// Filter for highways/roads OR buildings OR landuse OR water
let is_highway = tags.contains_key("highway");
let is_building = tags.contains_key("building");
let is_water = tags.get("natural").map(|v| v == "water").unwrap_or(false) ||
tags.get("waterway").map(|v| v == "riverbank").unwrap_or(false) ||
tags.get("landuse").map(|v| v == "basin").unwrap_or(false);
let is_landuse = tags.get("leisure").map(|v| v == "park" || v == "garden").unwrap_or(false) ||
tags.get("landuse").map(|v| v == "grass" || v == "forest" || v == "meadow").unwrap_or(false) ||
tags.get("natural").map(|v| v == "wood" || v == "scrub").unwrap_or(false);
if is_highway || is_building {
if is_highway || is_building || is_water || is_landuse {
let mut points = Vec::new();
// Resolve nodes
@@ -126,14 +135,42 @@ async fn main() -> Result<()> {
}
if is_building {
// inserted_buildings += 1; // Need to add this counter
let tags_clone = tags.clone();
let blob_clone = blob.clone();
let session = session.clone();
join_set.spawn(async move {
let _ = session.query(
"INSERT INTO map_data.buildings (zoom, tile_x, tile_y, id, tags, points) VALUES (?, ?, ?, ?, ?, ?)",
(10, x, y, id, tags, blob),
(10, x, y, id, tags_clone, blob_clone),
).await;
});
inserted_buildings += 1;
}
if is_water {
let tags_clone = tags.clone();
let blob_clone = blob.clone();
let session = session.clone();
join_set.spawn(async move {
let _ = session.query(
"INSERT INTO map_data.water (zoom, tile_x, tile_y, id, tags, points) VALUES (?, ?, ?, ?, ?, ?)",
(10, x, y, id, tags_clone, blob_clone),
).await;
});
inserted_water += 1;
}
if is_landuse {
let tags_clone = tags.clone();
let blob_clone = blob.clone();
let session = session.clone();
join_set.spawn(async move {
let _ = session.query(
"INSERT INTO map_data.landuse (zoom, tile_x, tile_y, id, tags, points) VALUES (?, ?, ?, ?, ?, ?)",
(10, x, y, id, tags_clone, blob_clone),
).await;
});
inserted_landuse += 1;
}
}
}
@@ -146,7 +183,8 @@ async fn main() -> Result<()> {
}
})?;
println!("Finished processing. Nodes: {}, Ways: {}. Inserted Nodes: {}, Inserted Ways: {}", node_count, way_count, inserted_nodes, inserted_ways);
println!("Finished processing. Nodes: {}, Ways: {}. Inserted Nodes: {}, Inserted Ways: {}, Buildings: {}, Water: {}, Landuse: {}",
node_count, way_count, inserted_nodes, inserted_ways, inserted_buildings, inserted_water, inserted_landuse);
println!("Waiting for pending inserts...");
while let Some(_) = join_set.join_next().await {}