commit 55f0f5f351689b5031b3e843fb8a86751798533c
parent c298f43d901567c1fa946974360169638ac613a8
Author: MTRNord <mtrnord1@gmail.com>
Date: Thu, 25 May 2023 20:19:33 +0200
Improve overall speed of the script by improving the changeset data loading and also fix a bug in the commit logic
Diffstat:
3 files changed, 118 insertions(+), 73 deletions(-)
diff --git a/src/git/mod.rs b/src/git/mod.rs
@@ -2,7 +2,7 @@ use std::{io::Write, path::Path};
use color_eyre::eyre::Result;
use git2::{Oid, Repository, Signature};
-use tracing::info;
+use tracing::{info, warn};
/// Initialize the git repository
///
@@ -102,7 +102,15 @@ pub fn commit(
} else {
Path::new(&file)
};
- index.add_path(path)?;
+ // TODO: I am tired to actually debug this so we just do a sanity check if the file exists
+ if file_path.exists() {
+ index.add_path(path)?;
+ } else {
+ warn!(
+ "File {} does not exist but was meant to be added",
+ path.to_str().unwrap()
+ );
+ }
}
for file in removed_files {
let file_path = Path::new(&file);
@@ -111,7 +119,10 @@ pub fn commit(
} else {
Path::new(&file)
};
- index.remove_path(path)?;
+ // We check if it was tracked before. If not we don't need to remove it
+ if index.get_path(path, 0).is_some() {
+ index.remove_path(path)?;
+ }
}
index.write()?;
index.write_tree()?
diff --git a/src/osm/changesets.rs b/src/osm/changesets.rs
@@ -6,7 +6,7 @@ use quick_xml::{
};
use std::{
borrow::Cow,
- collections::HashMap,
+ collections::{BTreeSet, HashMap},
convert::Infallible,
fs::File,
io::{BufReader, Write},
@@ -33,7 +33,8 @@ impl Changeset {
fn new_from_element(
reader: &mut Reader<BufReader<Decoder<'_, BufReader<File>>>>,
element: &BytesStart,
- ) -> Result<Self> {
+ changeset_list: &[u64],
+ ) -> Result<Option<Self>> {
let changeset_attributes: HashMap<String, String> = element
.attributes()
.filter_map(|attr_result| attr_result.ok())
@@ -67,6 +68,11 @@ impl Changeset {
//debug!("changeset_attributes: {:?}", changeset_attributes);
+ let id = changeset_attributes.get("id").unwrap().parse().unwrap();
+ if !changeset_list.contains(&id) {
+ return Ok(None);
+ }
+
let mut changeset = Changeset {
id: changeset_attributes.get("id").unwrap().parse().unwrap(),
created_at: changeset_attributes.get("created_at").unwrap().to_string(),
@@ -101,51 +107,55 @@ impl Changeset {
loop {
let event = reader.read_event_into(&mut new_buf)?;
- if let Event::End(ref e) = event {
- if e.name() == element.name() {
- break;
+ match event {
+ Event::End(ref e) => {
+ if e.name() == element.name() {
+ break;
+ }
}
- }
- if let Event::Start(ref e) = event {
- let name = e.name();
- if name == QName(b"tag") {
- let mut key = Cow::Borrowed("");
- let mut value = Cow::Borrowed("");
-
- for attr_result in element.attributes() {
- let a = attr_result?;
- match a.key.as_ref() {
- b"k" => key = a.decode_and_unescape_value(reader)?,
- b"v" => value = a.decode_and_unescape_value(reader)?,
- _ => (),
+ Event::Start(ref e) => {
+ let name = e.name();
+ if name == QName(b"tag") {
+ let mut key = Cow::Borrowed("");
+ let mut value = Cow::Borrowed("");
+
+ for attr_result in element.attributes() {
+ let a = attr_result?;
+ match a.key.as_ref() {
+ b"k" => key = a.decode_and_unescape_value(reader)?,
+ b"v" => value = a.decode_and_unescape_value(reader)?,
+ _ => (),
+ }
}
- }
- changeset.tags.insert(key.to_string(), value.to_string());
- } else {
- warn!("Unexpected tag: {:?}", name);
- }
- } else {
- if let Event::Text(ref text) = event {
- if text.borrow().starts_with(b"\n") {
- continue;
- }
- } else if let Event::End(ref e) = event {
- if e.name() == QName(b"tag") {
- continue;
+ changeset.tags.insert(key.to_string(), value.to_string());
+ } else {
+ warn!("Unexpected tag: {:?}", name);
+ //reader.read_to_end_into(e.name(), &mut new_buf);
}
}
- warn!("Unexpected event in changeset: {:?}", event);
- // Write the data to file for debugging
+ _ => {
+ if let Event::Text(ref text) = event {
+ if text.borrow().starts_with(b"\n") {
+ continue;
+ }
+ } else if let Event::End(ref e) = event {
+ if e.name() == QName(b"tag") {
+ continue;
+ }
+ }
+ warn!("Unexpected event in changeset: {:?}", event);
+ // Write the data to file for debugging
- let mut file = std::fs::File::create("debug.xml")?;
- file.write_all(&new_buf)?;
- file.sync_all()?;
+ let mut file = std::fs::File::create("debug.xml")?;
+ file.write_all(&new_buf)?;
+ file.sync_all()?;
+ }
}
new_buf = Vec::new();
}
- Ok(changeset)
+ Ok(Some(changeset))
}
}
@@ -170,23 +180,33 @@ pub fn parse_changeset(
changeset_data.expand_empty_elements(true);
let mut changesets = Vec::new();
+ let changeset_hashset = changeset_list.iter().cloned().collect::<BTreeSet<u64>>();
let mut buf = Vec::new();
// Parse the changeset file
info!("Parsing changeset file");
loop {
+ // If we already have all of them then break
+ // We compare the ids even if its a little more expensive
+ let ids_parsed = changesets
+ .iter()
+ .map(|c: &Changeset| c.id)
+ .collect::<BTreeSet<u64>>();
+ if changeset_hashset.is_subset(&ids_parsed) {
+ break;
+ }
+
let event = changeset_data.read_event_into(&mut buf)?;
match event {
Event::Start(element) => {
if let b"changeset" = element.name().as_ref() {
// TODO: What do we do in case of an error?
- let changeset = Changeset::new_from_element(changeset_data, &element);
+ let changeset =
+ Changeset::new_from_element(changeset_data, &element, changeset_list);
match changeset {
- Ok(changeset) => {
- if changeset_list.contains(&changeset.id) {
- changesets.push(changeset);
- }
+ Ok(Some(changeset)) => {
+ changesets.push(changeset);
}
Err(err) => {
error!(
@@ -194,8 +214,10 @@ pub fn parse_changeset(
&element, err
);
}
+ _ => {}
}
}
+ //changeset_data.read_to_end_into(element.name(), &mut buf);
}
Event::Eof => break, // exits the loop when reaching end of file
_ => (), // There are `Event` types not considered here
diff --git a/src/osm/osm_data.rs b/src/osm/osm_data.rs
@@ -469,6 +469,16 @@ pub enum OSMObject {
Relation(Relation),
}
+impl OSMObject {
+ pub fn id(&self) -> u64 {
+ match self {
+ OSMObject::Node(node) => node.id,
+ OSMObject::Way(way) => way.id,
+ OSMObject::Relation(relation) => relation.id,
+ }
+ }
+}
+
pub fn convert_objects_to_git(
repository: &Repository,
committer: &Signature,
@@ -542,7 +552,7 @@ pub fn convert_objects_to_git(
}
}
} else if name == QName(b"way") {
- let way = Way::new_from_element(&mut data, &e);
+ let way = Way::new_from_element(&mut data, e);
match way {
Ok(way) => created_objects.push(OSMObject::Way(way)),
Err(err) => {
@@ -553,7 +563,7 @@ pub fn convert_objects_to_git(
}
}
} else if name == QName(b"relation") {
- let relation = Relation::new_from_element(&mut data, &e);
+ let relation = Relation::new_from_element(&mut data, e);
match relation {
Ok(relation) => {
created_objects.push(OSMObject::Relation(relation))
@@ -595,7 +605,13 @@ pub fn convert_objects_to_git(
OSMObject::Relation(ref relation) => format!("{}.yaml", relation.id),
};
let object_file_path = repository_folder.join(object_file_name);
- let object_file = std::fs::File::create(object_file_path)?;
+
+ // We need to create the file
+ let object_file = OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .open(&object_file_path)?;
serde_yaml::to_writer(object_file, &object)?;
// Add the object to the list of created objects for the changeset based on the changeset id
@@ -627,7 +643,7 @@ pub fn convert_objects_to_git(
if let Event::Start(ref e) = event {
let name = e.name();
if name == QName(b"node") {
- let node = Node::new_from_element(&mut data, &e);
+ let node = Node::new_from_element(&mut data, e);
match node {
Ok(node) => deleted_objects.push(OSMObject::Node(node)),
Err(err) => {
@@ -638,7 +654,7 @@ pub fn convert_objects_to_git(
}
}
} else if name == QName(b"way") {
- let way = Way::new_from_element(&mut data, &e);
+ let way = Way::new_from_element(&mut data, e);
match way {
Ok(way) => deleted_objects.push(OSMObject::Way(way)),
Err(err) => {
@@ -649,7 +665,7 @@ pub fn convert_objects_to_git(
}
}
} else if name == QName(b"relation") {
- let relation = Relation::new_from_element(&mut data, &e);
+ let relation = Relation::new_from_element(&mut data, e);
match relation {
Ok(relation) => {
deleted_objects.push(OSMObject::Relation(relation))
@@ -779,7 +795,7 @@ pub fn convert_objects_to_git(
if let Event::Start(ref e) = event {
let name = e.name();
if name == QName(b"node") {
- let node = Node::new_from_element(&mut data, &e);
+ let node = Node::new_from_element(&mut data, e);
match node {
Ok(node) => deleted_objects.push(OSMObject::Node(node)),
Err(err) => {
@@ -790,7 +806,7 @@ pub fn convert_objects_to_git(
}
}
} else if name == QName(b"way") {
- let way = Way::new_from_element(&mut data, &e);
+ let way = Way::new_from_element(&mut data, e);
match way {
Ok(way) => deleted_objects.push(OSMObject::Way(way)),
Err(err) => {
@@ -801,7 +817,7 @@ pub fn convert_objects_to_git(
}
}
} else if name == QName(b"relation") {
- let relation = Relation::new_from_element(&mut data, &e);
+ let relation = Relation::new_from_element(&mut data, e);
match relation {
Ok(relation) => {
deleted_objects.push(OSMObject::Relation(relation))
@@ -858,20 +874,6 @@ pub fn convert_objects_to_git(
.entry(changeset)
.or_insert_with(Vec::new)
.push(object.clone());
- // Remove it from the list of created objects if it exists
- if let Some(index) = created_or_modified_objects_for_changeset
- .get_mut(&changeset)
- .and_then(|objects| {
- objects
- .iter()
- .position(|existing_object| *existing_object == object)
- })
- {
- created_or_modified_objects_for_changeset
- .get_mut(&changeset)
- .unwrap()
- .remove(index);
- }
}
}
_ => (),
@@ -895,6 +897,18 @@ pub fn convert_objects_to_git(
let mut last_highest_id = 0;
let mut changeset_path = String::new();
for changeset_file in changeset_files {
+ // Delete all objects by id that are in deleted_objects_for_changeset from created_or_modified_objects_for_changeset
+ let deleted_ids: Vec<u64> = deleted_objects_for_changeset
+ .values()
+ .flatten()
+ .map(|object| object.id())
+ .collect();
+ created_or_modified_objects_for_changeset
+ .iter_mut()
+ .for_each(|(_, objects)| {
+ objects.retain(|object| !deleted_ids.contains(&object.id()));
+ });
+
let changeset_file = changeset_file?;
let changeset_file_path = changeset_file.path();
let changeset_file_name = changeset_file_path.file_name().unwrap().to_str().unwrap();
@@ -914,6 +928,8 @@ pub fn convert_objects_to_git(
let changesets = parse_changeset(&mut uncompressed_data, &changeset_list)?;
+ info!("Generating commits for changesets");
+
for changeset_id in changeset_list {
// Find the changeset within the files of the cache
let changeset = find_changesets_in_cache(&changesets, changeset_id)?;
@@ -1032,11 +1048,7 @@ fn find_changesets_in_cache(
changesets: &[Changeset],
changeset_id: u64,
) -> Result<Option<&Changeset>> {
- let mut changeset = None;
-
- if changesets.iter().any(|c| c.id == changeset_id) {
- changeset = changesets.iter().find(|c| c.id == changeset_id);
- }
+ let changeset = changesets.iter().find(|c| c.id == changeset_id);
Ok(changeset)
}
