osm_data.rs (44146B)
1 use color_eyre::eyre::Result; 2 use flate2::bufread::GzDecoder; 3 use git2::{Repository, Signature, Time}; 4 use quick_xml::{ 5 events::{BytesStart, Event}, 6 name::QName, 7 Reader, 8 }; 9 use serde::{Deserialize, Serialize}; 10 use std::{ 11 borrow::Cow, 12 collections::BTreeMap, 13 convert::Infallible, 14 fs::{File, OpenOptions}, 15 io::{Read, Write}, 16 }; 17 use time::{format_description::well_known::Iso8601, OffsetDateTime}; 18 use tracing::{debug, error, info, warn}; 19 20 use crate::git::commit; 21 22 use super::changesets::{parse_changeset, uncompress_changeset_file, Changeset}; 23 24 const FILE_VERSION: &str = "0.1.0"; 25 26 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] 27 pub struct Node { 28 /// The id of the node. Saved as the file name. 29 #[serde(skip)] 30 pub id: u64, 31 #[serde(skip)] 32 pub changeset: u64, 33 #[serde(skip_serializing_if = "Option::is_none")] 34 pub file_generator: Option<String>, 35 pub file_version: String, 36 #[serde(skip_serializing_if = "Option::is_none")] 37 pub legacy_object_version: Option<String>, 38 pub lat: f64, 39 pub lon: f64, 40 #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] 41 pub tags: BTreeMap<String, String>, 42 } 43 impl Node { 44 fn new_from_element(reader: &mut Reader<&[u8]>, element: &BytesStart) -> Result<Self> { 45 let attributes: BTreeMap<String, String> = element 46 .attributes() 47 .filter_map(|attr_result| attr_result.ok()) 48 .map(|attr| { 49 let key = reader 50 .decoder() 51 .decode(attr.key.local_name().as_ref()) 52 .or_else(|err| { 53 dbg!( 54 "unable to read key in DefaultSettings attribute {:?}, utf8 error {:?}", 55 &attr, 56 err 57 ); 58 Ok::<Cow<'_, str>, Infallible>(std::borrow::Cow::from("")) 59 }) 60 .unwrap() 61 .to_string(); 62 let value = attr 63 .decode_and_unescape_value(reader) 64 .or_else(|err| { 65 dbg!( 66 "unable to read key in DefaultSettings attribute {:?}, utf8 error {:?}", 67 &attr, 68 err 69 ); 70 Ok::<Cow<'_, str>, Infallible>(std::borrow::Cow::from("")) 71 }) 72 .unwrap() 73 .to_string(); 74 (key, value) 75 }) 76 .collect(); 77 78 let mut node = Node { 79 id: attributes 80 .get("id") 81 .unwrap() 82 .parse::<u64>() 83 .expect("Unable to parse node id"), 84 changeset: attributes 85 .get("changeset") 86 .unwrap() 87 .parse::<u64>() 88 .expect("Unable to parse node changeset"), 89 file_generator: attributes.get("generator").map(|s| s.to_string()), 90 legacy_object_version: attributes.get("version").map(|s| s.to_string()), 91 lat: attributes 92 .get("lat") 93 .unwrap() 94 .parse::<f64>() 95 .expect("Unable to parse node lat"), 96 lon: attributes 97 .get("lon") 98 .unwrap() 99 .parse::<f64>() 100 .expect("Unable to parse node lon"), 101 tags: BTreeMap::new(), 102 file_version: FILE_VERSION.to_string(), 103 }; 104 105 let mut buf = Vec::new(); 106 loop { 107 let event = reader.read_event_into(&mut buf)?; 108 109 if let Event::End(ref e) = event { 110 if e.name() == element.name() { 111 break; 112 } 113 } 114 115 if let Event::Start(ref e) = event { 116 let name = e.name(); 117 if name == QName(b"tag") { 118 let mut key = Cow::Borrowed(""); 119 let mut value = Cow::Borrowed(""); 120 121 for attr_result in e.attributes() { 122 let a = attr_result?; 123 match a.key.as_ref() { 124 b"k" => key = a.decode_and_unescape_value(reader)?, 125 b"v" => value = a.decode_and_unescape_value(reader)?, 126 _ => (), 127 } 128 } 129 130 node.tags.insert(key.to_string(), value.to_string()); 131 } else { 132 warn!("Unexpected tag: {:?}", name); 133 } 134 reader.read_to_end(name)?; 135 } else { 136 if let Event::Text(ref text) = event { 137 if text.borrow().starts_with(b"\n") { 138 continue; 139 } 140 } else if let Event::End(ref e) = event { 141 if e.name() == QName(b"tag") { 142 continue; 143 } 144 } 145 warn!("Unexpected event in node: {:?}", event); 146 // Write the data to file for debugging 147 148 let mut file = std::fs::File::create("debug.xml")?; 149 file.write_all(&buf)?; 150 file.sync_all()?; 151 } 152 buf = Vec::new(); 153 } 154 155 Ok(node) 156 } 157 } 158 159 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] 160 pub struct Way { 161 /// The id of the node. Saved as the file name. 162 #[serde(skip)] 163 pub id: u64, 164 #[serde(skip)] 165 pub changeset: u64, 166 #[serde(skip_serializing_if = "Option::is_none")] 167 pub file_generator: Option<String>, 168 pub file_version: String, 169 #[serde(skip_serializing_if = "Option::is_none")] 170 pub legacy_object_version: Option<String>, 171 #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] 172 pub tags: BTreeMap<String, String>, 173 #[serde(default, skip_serializing_if = "Vec::is_empty")] 174 pub nodes: Vec<u64>, 175 } 176 177 impl Way { 178 fn new_from_element(reader: &mut Reader<&[u8]>, element: &BytesStart) -> Result<Self> { 179 let attributes: BTreeMap<String, String> = element 180 .attributes() 181 .filter_map(|attr_result| attr_result.ok()) 182 .map(|attr| { 183 let key = reader 184 .decoder() 185 .decode(attr.key.local_name().as_ref()) 186 .or_else(|err| { 187 dbg!( 188 "unable to read key in DefaultSettings attribute {:?}, utf8 error {:?}", 189 &attr, 190 err 191 ); 192 Ok::<Cow<'_, str>, Infallible>(std::borrow::Cow::from("")) 193 }) 194 .unwrap() 195 .to_string(); 196 let value = attr 197 .decode_and_unescape_value(reader) 198 .or_else(|err| { 199 dbg!( 200 "unable to read key in DefaultSettings attribute {:?}, utf8 error {:?}", 201 &attr, 202 err 203 ); 204 Ok::<Cow<'_, str>, Infallible>(std::borrow::Cow::from("")) 205 }) 206 .unwrap() 207 .to_string(); 208 (key, value) 209 }) 210 .collect(); 211 212 let mut way = Way { 213 id: attributes 214 .get("id") 215 .unwrap() 216 .parse::<u64>() 217 .expect("Unable to parse way id"), 218 changeset: attributes 219 .get("changeset") 220 .unwrap() 221 .parse::<u64>() 222 .expect("Unable to parse way changeset"), 223 file_generator: attributes.get("generator").map(|s| s.to_string()), 224 legacy_object_version: attributes.get("version").map(|s| s.to_string()), 225 tags: BTreeMap::new(), 226 nodes: Vec::new(), 227 file_version: FILE_VERSION.to_string(), 228 }; 229 230 let mut buf = Vec::new(); 231 loop { 232 let event = reader.read_event_into(&mut buf)?; 233 234 if let Event::End(ref e) = event { 235 if e.name() == element.name() { 236 break; 237 } 238 } 239 240 if let Event::Start(ref e) = event { 241 let name = e.name(); 242 if name == QName(b"tag") { 243 let mut key = Cow::Borrowed(""); 244 let mut value = Cow::Borrowed(""); 245 246 for attr_result in e.attributes() { 247 let a = attr_result?; 248 match a.key.as_ref() { 249 b"k" => key = a.decode_and_unescape_value(reader)?, 250 b"v" => value = a.decode_and_unescape_value(reader)?, 251 _ => (), 252 } 253 } 254 255 way.tags.insert(key.to_string(), value.to_string()); 256 } else if name == QName(b"nd") { 257 let mut ref_id = Cow::Borrowed(""); 258 259 for attr_result in e.attributes() { 260 let a = attr_result?; 261 if let b"ref" = a.key.as_ref() { 262 ref_id = a.decode_and_unescape_value(reader)?; 263 } 264 } 265 266 way.nodes.push( 267 ref_id 268 .to_string() 269 .parse::<u64>() 270 .expect("Unable to parse way node ref"), 271 ); 272 } else { 273 warn!("Unexpected tag: {:?}", name); 274 } 275 reader.read_to_end(name)?; 276 } else { 277 if let Event::Text(ref text) = event { 278 if text.borrow().starts_with(b"\n") { 279 continue; 280 } 281 } else if let Event::End(ref e) = event { 282 if e.name() == QName(b"tag") || e.name() == QName(b"nd") { 283 continue; 284 } 285 } 286 warn!("Unexpected event way: {:?}", event); 287 // Write the data to file for debugging 288 289 let mut file = std::fs::File::create("debug.xml")?; 290 file.write_all(&buf)?; 291 file.sync_all()?; 292 } 293 buf = Vec::new(); 294 } 295 296 Ok(way) 297 } 298 } 299 300 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] 301 pub struct RelationMember { 302 #[serde(rename = "type")] 303 pub r#type: String, 304 #[serde(rename = "ref")] 305 pub ref_id: u64, 306 #[serde(skip_serializing_if = "Option::is_none")] 307 pub role: Option<String>, 308 } 309 310 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] 311 pub struct Relation { 312 /// The id of the node. Saved as the file name. 313 #[serde(skip)] 314 pub id: u64, 315 #[serde(skip)] 316 pub changeset: u64, 317 #[serde(skip_serializing_if = "Option::is_none")] 318 pub file_generator: Option<String>, 319 pub file_version: String, 320 #[serde(skip_serializing_if = "Option::is_none")] 321 pub legacy_object_version: Option<String>, 322 #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] 323 pub tags: BTreeMap<String, String>, 324 #[serde(default, skip_serializing_if = "Vec::is_empty")] 325 pub member: Vec<RelationMember>, 326 } 327 328 impl Relation { 329 fn new_from_element(reader: &mut Reader<&[u8]>, element: &BytesStart) -> Result<Self> { 330 let attributes: BTreeMap<String, String> = element 331 .attributes() 332 .filter_map(|attr_result| attr_result.ok()) 333 .map(|attr| { 334 let key = reader 335 .decoder() 336 .decode(attr.key.local_name().as_ref()) 337 .or_else(|err| { 338 dbg!( 339 "unable to read key in DefaultSettings attribute {:?}, utf8 error {:?}", 340 &attr, 341 err 342 ); 343 Ok::<Cow<'_, str>, Infallible>(std::borrow::Cow::from("")) 344 }) 345 .unwrap() 346 .to_string(); 347 let value = attr 348 .decode_and_unescape_value(reader) 349 .or_else(|err| { 350 dbg!( 351 "unable to read key in DefaultSettings attribute {:?}, utf8 error {:?}", 352 &attr, 353 err 354 ); 355 Ok::<Cow<'_, str>, Infallible>(std::borrow::Cow::from("")) 356 }) 357 .unwrap() 358 .to_string(); 359 (key, value) 360 }) 361 .collect(); 362 363 let mut relation = Relation { 364 id: attributes 365 .get("id") 366 .unwrap() 367 .parse::<u64>() 368 .expect("Unable to parse way id"), 369 changeset: attributes 370 .get("changeset") 371 .unwrap() 372 .parse::<u64>() 373 .expect("Unable to parse way changeset"), 374 file_generator: attributes.get("generator").map(|s| s.to_string()), 375 legacy_object_version: attributes.get("version").map(|s| s.to_string()), 376 tags: BTreeMap::new(), 377 member: Vec::new(), 378 file_version: FILE_VERSION.to_string(), 379 }; 380 381 let mut buf = Vec::new(); 382 loop { 383 let event = reader.read_event_into(&mut buf)?; 384 385 if let Event::End(ref e) = event { 386 if e.name() == element.name() { 387 break; 388 } 389 } 390 391 if let Event::Start(ref e) = event { 392 let name = e.name(); 393 if name == QName(b"tag") { 394 let mut key = Cow::Borrowed(""); 395 let mut value = Cow::Borrowed(""); 396 397 for attr_result in e.attributes() { 398 let a = attr_result?; 399 match a.key.as_ref() { 400 b"k" => key = a.decode_and_unescape_value(reader)?, 401 b"v" => value = a.decode_and_unescape_value(reader)?, 402 _ => (), 403 } 404 } 405 406 relation.tags.insert(key.to_string(), value.to_string()); 407 } else if name == QName(b"member") { 408 let mut ref_id = Cow::Borrowed(""); 409 let mut r#type = Cow::Borrowed(""); 410 let mut role = Cow::Borrowed(""); 411 412 for attr_result in e.attributes() { 413 let a = attr_result?; 414 match a.key.as_ref() { 415 b"ref" => ref_id = a.decode_and_unescape_value(reader)?, 416 b"type" => r#type = a.decode_and_unescape_value(reader)?, 417 b"role" => role = a.decode_and_unescape_value(reader)?, 418 _ => (), 419 } 420 } 421 422 let normalized_role = if role.is_empty() { 423 None 424 } else { 425 Some(role.to_string()) 426 }; 427 428 relation.member.push(RelationMember { 429 r#type: r#type.to_string(), 430 ref_id: ref_id 431 .to_string() 432 .parse::<u64>() 433 .expect("Unable to parse relation member ref"), 434 role: normalized_role, 435 }); 436 } else { 437 warn!("Unexpected tag: {:?}", name); 438 } 439 reader.read_to_end(name)?; 440 } else { 441 if let Event::Text(ref text) = event { 442 if text.borrow().starts_with(b"\n") { 443 continue; 444 } 445 } else if let Event::End(ref e) = event { 446 if e.name() == QName(b"tag") || e.name() == QName(b"member") { 447 continue; 448 } 449 } 450 warn!("Unexpected event in Relation: {:?}", event); 451 // Write the data to file for debugging 452 453 let mut file = std::fs::File::create("debug.xml")?; 454 file.write_all(&buf)?; 455 file.sync_all()?; 456 } 457 buf = Vec::new(); 458 } 459 460 Ok(relation) 461 } 462 } 463 464 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] 465 #[serde(tag = "type")] 466 pub enum OSMObject { 467 Node(Node), 468 Way(Way), 469 Relation(Relation), 470 } 471 472 impl OSMObject { 473 pub fn id(&self) -> u64 { 474 match self { 475 OSMObject::Node(node) => node.id, 476 OSMObject::Way(way) => way.id, 477 OSMObject::Relation(relation) => relation.id, 478 } 479 } 480 } 481 482 pub fn convert_objects_to_git( 483 repository: &Repository, 484 committer: &Signature, 485 data: &[u8], 486 changesets_location: &str, 487 ) -> Result<()> { 488 // If the file is empty we skip it 489 if data.is_empty() { 490 return Ok(()); 491 } 492 493 // Decompress the changeset file 494 let mut data_reader = GzDecoder::new(data); 495 let mut file_data = String::new(); 496 if let Err(e) = data_reader.read_to_string(&mut file_data) { 497 error!("Unable to decompress data file: {:?}. Moving on", e); 498 return Ok(()); 499 } 500 debug!("Data file decompressed. Size: {}", file_data.len()); 501 502 // If the file is empty we skip it 503 if file_data.is_empty() { 504 return Ok(()); 505 } 506 507 info!("Parsing data file"); 508 509 let mut data = Reader::from_str(&file_data); 510 511 // == Handling empty elements == 512 // To simply our processing code 513 // we want the same events for empty elements, like: 514 // <DefaultSettings Language="es" Greeting="HELLO"/> 515 // <Text/> 516 data.expand_empty_elements(true); 517 518 let mut buf = Vec::new(); 519 let mut skip_buf = Vec::new(); 520 let mut created_or_modified_objects_for_changeset = BTreeMap::new(); 521 let mut deleted_objects_for_changeset = BTreeMap::new(); 522 523 loop { 524 let event: Event = data.read_event_into(&mut buf)?; 525 match event { 526 Event::Start(element) => match element.name().as_ref() { 527 b"create" => { 528 // TODO: What do we do in case of an error? 529 530 let mut created_objects = Vec::new(); 531 532 loop { 533 let event = data.read_event_into(&mut skip_buf)?; 534 535 if let Event::End(ref e) = event { 536 if e.name() == element.name() { 537 break; 538 } 539 } 540 541 if let Event::Start(ref e) = event { 542 let name = e.name(); 543 if name == QName(b"node") { 544 let node = Node::new_from_element(&mut data, e); 545 match node { 546 Ok(node) => created_objects.push(OSMObject::Node(node)), 547 Err(err) => { 548 error!( 549 "unable to read node element {:?}, utf8 error {:?}", 550 &e, err 551 ); 552 } 553 } 554 } else if name == QName(b"way") { 555 let way = Way::new_from_element(&mut data, e); 556 match way { 557 Ok(way) => created_objects.push(OSMObject::Way(way)), 558 Err(err) => { 559 error!( 560 "unable to read way element {:?}, utf8 error {:?}", 561 &e, err 562 ); 563 } 564 } 565 } else if name == QName(b"relation") { 566 let relation = Relation::new_from_element(&mut data, e); 567 match relation { 568 Ok(relation) => { 569 created_objects.push(OSMObject::Relation(relation)) 570 } 571 Err(err) => { 572 error!( 573 "unable to read relation element {:?}, utf8 error {:?}", 574 &e, err 575 ); 576 } 577 } 578 } else { 579 warn!("Unexpected tag: {:?}", name); 580 data.read_to_end(name)?; 581 } 582 } else { 583 if let Event::Text(ref text) = event { 584 if text.borrow().starts_with(b"\n") { 585 continue; 586 } 587 } 588 warn!("Unexpected event in create: {:?}", event); 589 // Write the data to file for debugging 590 591 let mut file = std::fs::File::create("debug.xml")?; 592 file.write_all(file_data.as_bytes())?; 593 file.sync_all()?; 594 } 595 skip_buf = Vec::new(); 596 } 597 598 // write the objects to the git repo as yaml files 599 let repository_folder = repository.path().parent().unwrap(); 600 // TODO: We should chunk the world and split it into folders... Otherwise good luck 601 for object in created_objects { 602 let object_file_name = match object { 603 OSMObject::Node(ref node) => format!("{}.yaml", node.id), 604 OSMObject::Way(ref way) => format!("{}.yaml", way.id), 605 OSMObject::Relation(ref relation) => format!("{}.yaml", relation.id), 606 }; 607 let object_file_path = repository_folder.join(object_file_name); 608 609 // We need to create the file 610 let object_file = OpenOptions::new() 611 .read(true) 612 .write(true) 613 .create(true) 614 .open(&object_file_path)?; 615 serde_yaml::to_writer(object_file, &object)?; 616 617 // Add the object to the list of created objects for the changeset based on the changeset id 618 let changeset = match object { 619 OSMObject::Node(ref node) => node.changeset, 620 OSMObject::Way(ref way) => way.changeset, 621 OSMObject::Relation(ref relation) => relation.changeset, 622 }; 623 created_or_modified_objects_for_changeset 624 .entry(changeset) 625 .or_insert_with(Vec::new) 626 .push(object); 627 } 628 } 629 b"modify" => { 630 // TODO: What do we do in case of an error? 631 632 let mut deleted_objects = Vec::new(); 633 634 loop { 635 let event = data.read_event_into(&mut skip_buf)?; 636 637 if let Event::End(ref e) = event { 638 if e.name() == element.name() { 639 break; 640 } 641 } 642 643 if let Event::Start(ref e) = event { 644 let name = e.name(); 645 if name == QName(b"node") { 646 let node = Node::new_from_element(&mut data, e); 647 match node { 648 Ok(node) => deleted_objects.push(OSMObject::Node(node)), 649 Err(err) => { 650 error!( 651 "unable to read node element {:?}, utf8 error {:?}", 652 &e, err 653 ); 654 } 655 } 656 } else if name == QName(b"way") { 657 let way = Way::new_from_element(&mut data, e); 658 match way { 659 Ok(way) => deleted_objects.push(OSMObject::Way(way)), 660 Err(err) => { 661 error!( 662 "unable to read way element {:?}, utf8 error {:?}", 663 &e, err 664 ); 665 } 666 } 667 } else if name == QName(b"relation") { 668 let relation = Relation::new_from_element(&mut data, e); 669 match relation { 670 Ok(relation) => { 671 deleted_objects.push(OSMObject::Relation(relation)) 672 } 673 Err(err) => { 674 error!( 675 "unable to read relation element {:?}, utf8 error {:?}", 676 &e, err 677 ); 678 } 679 } 680 } else { 681 warn!("Unexpected tag: {:?}", name); 682 data.read_to_end(name)?; 683 } 684 } else { 685 if let Event::Text(ref text) = event { 686 if text.borrow().starts_with(b"\n") { 687 continue; 688 } 689 } 690 warn!("Unexpected event in create: {:?}", event); 691 // Write the data to file for debugging 692 693 let mut file = std::fs::File::create("debug.xml")?; 694 file.write_all(file_data.as_bytes())?; 695 file.sync_all()?; 696 } 697 skip_buf = Vec::new(); 698 } 699 700 // write the objects to the git repo as yaml files 701 let repository_folder = repository.path().parent().unwrap(); 702 for object in deleted_objects { 703 let object_file_name = match object { 704 OSMObject::Node(ref node) => format!("{}.yaml", node.id), 705 OSMObject::Way(ref way) => format!("{}.yaml", way.id), 706 OSMObject::Relation(ref relation) => format!("{}.yaml", relation.id), 707 }; 708 let object_file_path = repository_folder.join(object_file_name); 709 // Change the file according to the changeset 710 711 // If we got the file we open it otherwise we create a new object 712 if !object_file_path.exists() { 713 // We need to create the file 714 let object_file = OpenOptions::new() 715 .read(true) 716 .write(true) 717 .create(true) 718 .open(&object_file_path)?; 719 serde_yaml::to_writer(object_file, &object)?; 720 } 721 let mut object_file = 722 OpenOptions::new().read(true).open(&object_file_path)?; 723 724 let mut file_object: OSMObject = serde_yaml::from_reader(&mut object_file)?; 725 726 match object { 727 OSMObject::Node(ref node) => { 728 if let OSMObject::Node(ref mut file_node) = file_object { 729 file_node.changeset = node.changeset; 730 file_node.file_generator = node.file_generator.clone(); 731 file_node.file_version = node.file_version.clone(); 732 file_node.legacy_object_version = 733 node.legacy_object_version.clone(); 734 file_node.lat = node.lat; 735 file_node.lon = node.lon; 736 file_node.tags = node.tags.clone(); 737 } 738 } 739 OSMObject::Way(ref way) => { 740 if let OSMObject::Way(ref mut file_way) = file_object { 741 file_way.changeset = way.changeset; 742 file_way.file_generator = way.file_generator.clone(); 743 file_way.file_version = way.file_version.clone(); 744 file_way.legacy_object_version = 745 way.legacy_object_version.clone(); 746 file_way.tags = way.tags.clone(); 747 file_way.nodes = way.nodes.clone(); 748 } 749 } 750 OSMObject::Relation(ref relation) => { 751 if let OSMObject::Relation(ref mut file_relation) = file_object { 752 file_relation.changeset = relation.changeset; 753 file_relation.file_generator = relation.file_generator.clone(); 754 file_relation.file_version = relation.file_version.clone(); 755 file_relation.legacy_object_version = 756 relation.legacy_object_version.clone(); 757 file_relation.tags = relation.tags.clone(); 758 file_relation.member = relation.member.clone(); 759 } 760 } 761 } 762 let object_file = OpenOptions::new() 763 .read(true) 764 .write(true) 765 .truncate(true) 766 .open(object_file_path)?; 767 serde_yaml::to_writer(object_file, &object)?; 768 // Add the object to the list of created objects for the changeset based on the changeset id 769 let changeset = match object { 770 OSMObject::Node(ref node) => node.changeset, 771 OSMObject::Way(ref way) => way.changeset, 772 OSMObject::Relation(ref relation) => relation.changeset, 773 }; 774 775 created_or_modified_objects_for_changeset 776 .entry(changeset) 777 .or_insert_with(Vec::new) 778 .push(object); 779 } 780 } 781 b"delete" => { 782 // TODO: What do we do in case of an error? 783 784 let mut deleted_objects = Vec::new(); 785 786 loop { 787 let event = data.read_event_into(&mut skip_buf)?; 788 789 if let Event::End(ref e) = event { 790 if e.name() == element.name() { 791 break; 792 } 793 } 794 795 if let Event::Start(ref e) = event { 796 let name = e.name(); 797 if name == QName(b"node") { 798 let node = Node::new_from_element(&mut data, e); 799 match node { 800 Ok(node) => deleted_objects.push(OSMObject::Node(node)), 801 Err(err) => { 802 error!( 803 "unable to read node element {:?}, utf8 error {:?}", 804 &e, err 805 ); 806 } 807 } 808 } else if name == QName(b"way") { 809 let way = Way::new_from_element(&mut data, e); 810 match way { 811 Ok(way) => deleted_objects.push(OSMObject::Way(way)), 812 Err(err) => { 813 error!( 814 "unable to read way element {:?}, utf8 error {:?}", 815 &e, err 816 ); 817 } 818 } 819 } else if name == QName(b"relation") { 820 let relation = Relation::new_from_element(&mut data, e); 821 match relation { 822 Ok(relation) => { 823 deleted_objects.push(OSMObject::Relation(relation)) 824 } 825 Err(err) => { 826 error!( 827 "unable to read relation element {:?}, utf8 error {:?}", 828 &e, err 829 ); 830 } 831 } 832 } else { 833 warn!("Unexpected tag: {:?}", name); 834 data.read_to_end(name)?; 835 } 836 } else { 837 if let Event::Text(ref text) = event { 838 if text.borrow().starts_with(b"\n") { 839 continue; 840 } 841 } 842 warn!("Unexpected event in create: {:?}", event); 843 // Write the data to file for debugging 844 845 let mut file = std::fs::File::create("debug.xml")?; 846 file.write_all(file_data.as_bytes())?; 847 file.sync_all()?; 848 } 849 skip_buf = Vec::new(); 850 } 851 852 // write the objects to the git repo as yaml files 853 let repository_folder = repository.path().parent().unwrap(); 854 for object in deleted_objects { 855 let object_file_name = match object { 856 OSMObject::Node(ref node) => format!("{}.yaml", node.id), 857 OSMObject::Way(ref way) => format!("{}.yaml", way.id), 858 OSMObject::Relation(ref relation) => format!("{}.yaml", relation.id), 859 }; 860 let object_file_path = repository_folder.join(object_file_name); 861 862 // Delete the file if it exists 863 if object_file_path.exists() { 864 std::fs::remove_file(object_file_path)?; 865 } 866 867 // Add the object to the list of created objects for the changeset based on the changeset id 868 let changeset = match object { 869 OSMObject::Node(ref node) => node.changeset, 870 OSMObject::Way(ref way) => way.changeset, 871 OSMObject::Relation(ref relation) => relation.changeset, 872 }; 873 deleted_objects_for_changeset 874 .entry(changeset) 875 .or_insert_with(Vec::new) 876 .push(object.clone()); 877 } 878 } 879 _ => (), 880 }, 881 Event::Eof => break, // exits the loop when reaching end of file 882 _ => (), // There are `Event` types not considered here 883 } 884 buf = Vec::new(); 885 } 886 887 // For all the objects changed apply the changesets as commits 888 // Get changeset list from BTreeMaps 889 let changeset_list: Vec<u64> = created_or_modified_objects_for_changeset 890 .keys() 891 .chain(deleted_objects_for_changeset.keys()) 892 .copied() 893 .collect(); 894 895 // Find latest changeset file (highest number in filename after "changesets-" and before ".osm.zst") 896 let changeset_files = std::fs::read_dir(changesets_location)?; 897 let mut last_highest_id = 0; 898 let mut changeset_path = String::new(); 899 for changeset_file in changeset_files { 900 // Delete all objects by id that are in deleted_objects_for_changeset from created_or_modified_objects_for_changeset 901 let deleted_ids: Vec<u64> = deleted_objects_for_changeset 902 .values() 903 .flatten() 904 .map(|object| object.id()) 905 .collect(); 906 created_or_modified_objects_for_changeset 907 .iter_mut() 908 .for_each(|(_, objects)| { 909 objects.retain(|object| !deleted_ids.contains(&object.id())); 910 }); 911 912 let changeset_file = changeset_file?; 913 let changeset_file_path = changeset_file.path(); 914 let changeset_file_name = changeset_file_path.file_name().unwrap().to_str().unwrap(); 915 let changeset_file_name = changeset_file_name.trim_end_matches(".osm.zst"); 916 let changeset_file_name = changeset_file_name.trim_start_matches("changesets-"); 917 let changeset_file_name = changeset_file_name.parse::<u64>(); 918 if let Ok(changeset_file_name) = changeset_file_name { 919 if changeset_file_name > last_highest_id { 920 last_highest_id = changeset_file_name; 921 changeset_path = changeset_file_path.to_str().unwrap().to_string(); 922 } 923 } 924 } 925 926 let changeset_file = File::open(changeset_path)?; 927 let mut uncompressed_data = uncompress_changeset_file(changeset_file); 928 929 let changesets = parse_changeset(&mut uncompressed_data, &changeset_list)?; 930 931 info!("Generating commits for changesets"); 932 933 for changeset_id in changeset_list { 934 // Find the changeset within the files of the cache 935 let changeset = find_changesets_in_cache(&changesets, changeset_id)?; 936 937 if changeset.is_none() { 938 warn!("Unable to find changeset {:?}", changeset_id); 939 continue; 940 } 941 942 if let Some(changeset) = changeset { 943 // Get comment tag if it exists and trim it 944 let comment = changeset 945 .tags 946 .get("comment") 947 .map(|s| s.trim()) 948 .unwrap_or(""); 949 950 // Parse changeset time (ISO 8601) to git time (seconds since epoch) with offset 0 (UTC) using `time` 951 let changeset_time = changeset 952 .closed_at 953 .clone() 954 .unwrap_or(changeset.created_at.clone()); 955 let commit_time = 956 OffsetDateTime::parse(changeset_time.as_str(), &Iso8601::DEFAULT)?.unix_timestamp(); 957 958 let author = git2::Signature::new( 959 &changeset.user, 960 &format!("{}@osm", changeset.user), 961 &Time::new(commit_time, 0), 962 ) 963 .expect("Unable to create author signature"); 964 965 let repository_folder = repository.path().parent().unwrap(); 966 967 let added_or_changed_files = created_or_modified_objects_for_changeset 968 .get(&changeset.id) 969 .unwrap_or(&Vec::new()) 970 .iter() 971 .map(|object| match object { 972 OSMObject::Node(ref node) => { 973 repository_folder.join(format!("{}.yaml", node.id)) 974 } 975 OSMObject::Way(ref way) => repository_folder.join(format!("{}.yaml", way.id)), 976 OSMObject::Relation(ref relation) => { 977 repository_folder.join(format!("{}.yaml", relation.id)) 978 } 979 }) 980 .map(|path| path.to_string_lossy().to_string()) 981 .collect::<Vec<String>>(); 982 983 let removed_files = deleted_objects_for_changeset 984 .get(&changeset.id) 985 .unwrap_or(&Vec::new()) 986 .iter() 987 .map(|object| match object { 988 OSMObject::Node(ref node) => { 989 repository_folder.join(format!("{}.yaml", node.id)) 990 } 991 OSMObject::Way(ref way) => repository_folder.join(format!("{}.yaml", way.id)), 992 OSMObject::Relation(ref relation) => { 993 repository_folder.join(format!("{}.yaml", relation.id)) 994 } 995 }) 996 .map(|path| path.to_string_lossy().to_string()) 997 .collect::<Vec<String>>(); 998 999 let oid = commit( 1000 repository, 1001 added_or_changed_files, 1002 removed_files, 1003 comment, 1004 &author, 1005 committer, 1006 )?; 1007 1008 // Convert tags to "Key: Value" strings separated by newlines for the note 1009 let note = changeset 1010 .tags 1011 .iter() 1012 .filter_map(|(key, value)| { 1013 if key.trim().is_empty() { 1014 None 1015 } else { 1016 Some((key, value)) 1017 } 1018 }) 1019 .map(|(key, value)| format!("{}: {}", key, value)) 1020 .collect::<Vec<String>>() 1021 .join("\n"); 1022 1023 // Add the id of the changeset to the note 1024 let note = if note.is_empty() { 1025 format!("Legacy Changeset ID: {}", changeset.id) 1026 } else { 1027 format!("Legacy Changeset ID: {}\n{}", changeset.id, note) 1028 }; 1029 1030 repository.note(&author, committer, None, oid, ¬e, false)?; 1031 } 1032 } 1033 1034 Ok(()) 1035 } 1036 1037 /// Scans the files in the cache folder and returns the requested changeset 1038 /// 1039 /// # Arguments 1040 /// 1041 /// * `cache_folder` - The folder where the changesets are stored 1042 /// * `changeset_id` - The id of the changeset to find 1043 /// 1044 /// # Returns 1045 /// 1046 /// The changeset if found 1047 fn find_changesets_in_cache( 1048 changesets: &[Changeset], 1049 changeset_id: u64, 1050 ) -> Result<Option<&Changeset>> { 1051 let changeset = changesets.iter().find(|c| c.id == changeset_id); 1052 1053 Ok(changeset) 1054 }