osm-git

A WIP POC based on the idea from https://blog.andygol.co.ua/en/2023/05/07/osm-2-0-api-using-git/
git clone git://archive.git.mtrnord.blog/MTRNord/osm-git.git
Log | Files | Refs | LICENSE

commit 47198d5d6b1fa5db80778b07e8cec69a6f87f588
Author: MTRNord <mtrnord1@gmail.com>
Date:   Mon, 22 May 2023 02:12:39 +0200

Initial Version of the osm-changeset to git conversion tool

Diffstat:
A.gitignore | 5+++++
ACargo.lock | 1878+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
ACargo.toml | 21+++++++++++++++++++++
ALICENSE | 661+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Asrc/git/mod.rs | 116+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Asrc/main.rs | 279+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Asrc/osm/changesets.rs | 197+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Asrc/osm/mod.rs | 2++
Asrc/osm/osm_data.rs | 927+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Atemplates/README.md | 11+++++++++++
10 files changed, 4097 insertions(+), 0 deletions(-)

diff --git a/.gitignore b/.gitignore @@ -0,0 +1,4 @@ +/target +/osm-git-repo +/debug.xml +/cache +\ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock @@ -0,0 +1,1878 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "anstream" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is-terminal", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" + +[[package]] +name = "anstyle-parse" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "anstyle-wincon" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +dependencies = [ + "anstyle", + "windows-sys 0.48.0", +] + +[[package]] +name = "async-compression" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0122885821398cc923ece939e24d1056a2384ee719432397fa9db87230ff11" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-trait" +version = "0.1.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.16", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backtrace" +version = "0.3.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide 0.6.2", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bumpalo" +version = "3.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c6ed94e98ecff0c12dd1b04c15ec0d7d9458ca8fe806cea6f12954efe74c63b" + +[[package]] +name = "bytes" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +dependencies = [ + "jobserver", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93aae7a4192245f70fe75dd9157fc7b4a5bf53e88d30bd4396f7d8f9284d5acc" +dependencies = [ + "clap_builder", + "clap_derive", + "once_cell", +] + +[[package]] +name = "clap_builder" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f423e341edefb78c9caba2d9c7f7687d0e72e89df3ce3394554754393ac3990" +dependencies = [ + "anstream", + "anstyle", + "bitflags", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "191d9573962933b4027f932c600cd252ce27a8ad5979418fe78e43c07996f27b" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.16", +] + +[[package]] +name = "clap_lex" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" + +[[package]] +name = "color-eyre" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a667583cca8c4f8436db8de46ea8233c42a7d9ae424a82d338f2e4675229204" +dependencies = [ + "backtrace", + "color-spantrace", + "eyre", + "indenter", + "once_cell", + "owo-colors", + "tracing-error", +] + +[[package]] +name = "color-spantrace" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ba75b3d9449ecdccb27ecbc479fdc0b87fa2dd43d2f8298f9bf0e59aacc8dce" +dependencies = [ + "once_cell", + "owo-colors", + "tracing-core", + "tracing-error", +] + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "data-encoding" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" + +[[package]] +name = "encoding_rs" +version = "0.8.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "enum-as-inner" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "errno" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "eyre" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c2b6b5a29c02cdc822728b7d7b8ae1bab3e3b05d44522770ddd49722eeac7eb" +dependencies = [ + "indenter", + "once_cell", +] + +[[package]] +name = "flate2" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +dependencies = [ + "crc32fast", + "miniz_oxide 0.7.1", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" + +[[package]] +name = "futures-io" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" + +[[package]] +name = "futures-macro" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.16", +] + +[[package]] +name = "futures-sink" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" + +[[package]] +name = "futures-task" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" + +[[package]] +name = "futures-util" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +dependencies = [ + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" + +[[package]] +name = "git2" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b7905cdfe33d31a88bb2e8419ddd054451f5432d1da9eaf2ac7804ee1ea12d5" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "openssl-probe", + "openssl-sys", + "url", +] + +[[package]] +name = "h2" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi", +] + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "hyper" +version = "0.14.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7" +dependencies = [ + "http", + "hyper", + "rustls", + "tokio", + "tokio-rustls", +] + +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indenter" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +dependencies = [ + "hermit-abi 0.3.1", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "ipconfig" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd302af1b90f2463a98fa5ad469fc212c8e3175a41c3068601bfa2727591c5be" +dependencies = [ + "socket2", + "widestring", + "winapi", + "winreg", +] + +[[package]] +name = "ipnet" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" + +[[package]] +name = "is-terminal" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +dependencies = [ + "hermit-abi 0.3.1", + "io-lifetimes", + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "itoa" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + +[[package]] +name = "jobserver" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.144" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" + +[[package]] +name = "libgit2-sys" +version = "0.15.1+1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb4577bde8cdfc7d6a2a4bcb7b049598597de33ffd337276e9c7db6cd4a2cee7" +dependencies = [ + "cc", + "libc", + "libssh2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", +] + +[[package]] +name = "libssh2-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dc8a030b787e2119a731f1951d6a773e2280c660f8ec4b0f5e1505a386e71ee" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ee889ecc9568871456d42f603d6a0ce59ff328d291063a45cbdf0036baf6db" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + +[[package]] +name = "matches" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +dependencies = [ + "adler", +] + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.45.0", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "num_cpus" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +dependencies = [ + "hermit-abi 0.2.6", + "libc", +] + +[[package]] +name = "object" +version = "0.30.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e17f59264b2809d77ae94f0e1ebabc434773f370d6ca667bd223ea10e06cc7e" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "osm-git" +version = "0.1.0" +dependencies = [ + "bytes", + "clap", + "color-eyre", + "flate2", + "git2", + "quick-xml", + "reqwest", + "serde", + "serde_yaml", + "time", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "owo-colors" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys 0.45.0", +] + +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa1fb82fc0c281dd9671101b66b771ebbe1eaf967b96ac8740dcba4b70005ca8" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quick-xml" +version = "0.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce5e73202a820a31f8a0ee32ada5e21029c81fd9e3ebf668a40832e4219d9d1" +dependencies = [ + "encoding_rs", + "memchr", + "tokio", +] + +[[package]] +name = "quote" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "reqwest" +version = "0.11.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" +dependencies = [ + "async-compression", + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-rustls", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-rustls", + "tokio-util", + "tower-service", + "trust-dns-resolver", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", + "winreg", +] + +[[package]] +name = "resolv-conf" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" +dependencies = [ + "hostname", + "quick-error", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustix" +version = "0.37.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustls" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c911ba11bc8433e811ce56fde130ccf32f5127cab0e0194e9c68c5a5b671791e" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +dependencies = [ + "base64", +] + +[[package]] +name = "rustls-webpki" +version = "0.100.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "serde" +version = "1.0.163" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.163" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.16", +] + +[[package]] +name = "serde_json" +version = "1.0.96" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9d684e3ec7de3bf5466b32bd75303ac16f0736426e5a4e0d6e489559ce1249c" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.16", +] + +[[package]] +name = "thread_local" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "time" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3403384eaacbca9923fa06940178ac13e4edb725486d70e8e15881d0c836cc" +dependencies = [ + "itoa", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" + +[[package]] +name = "time-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" +dependencies = [ + "time-core", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105" +dependencies = [ + "autocfg", + "bytes", + "libc", + "mio", + "num_cpus", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-macros" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.16", +] + +[[package]] +name = "tokio-rustls" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0d409377ff5b1e3ca6437aa86c1eb7d40c134bfec254e44c830defa92669db5" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.16", +] + +[[package]] +name = "tracing-core" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-error" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" +dependencies = [ + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "tracing-log" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "trust-dns-proto" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f7f83d1e4a0e4358ac54c5c3681e5d7da5efc5a7a632c90bb6d6669ddd9bc26" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna 0.2.3", + "ipnet", + "lazy_static", + "rand", + "smallvec", + "thiserror", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aff21aa4dcefb0a1afbfac26deb0adc93888c7d295fb63ab273ef276ba2b7cfe" +dependencies = [ + "cfg-if", + "futures-util", + "ipconfig", + "lazy_static", + "lru-cache", + "parking_lot", + "resolv-conf", + "smallvec", + "thiserror", + "tokio", + "tracing", + "trust-dns-proto", +] + +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unsafe-libyaml" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1865806a559042e51ab5414598446a5871b561d21b6764f2eabb0dd481d880a6" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna 0.3.0", + "percent-encoding", +] + +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.16", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d1985d03709c53167ce907ff394f5316aa22cb4e12761295c5dc57dacb6297e" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.16", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" + +[[package]] +name = "wasm-streams" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bdd9ef4e984da1187bf8110c5cf5b845fbc87a23602cdf912386a76fcd3a7c2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +dependencies = [ + "webpki", +] + +[[package]] +name = "widestring" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17882f045410753661207383517a6f62ec3dbeb6a4ed2acce01f0728238d1983" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] diff --git a/Cargo.toml b/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "osm-git" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +bytes = "1.4.0" +clap = { version = "4.3.0", features = ["derive"] } +color-eyre = "0.6.2" +flate2 = "1.0.26" +git2 = "0.17.1" +quick-xml = { version = "0.28.2", features = ["async-tokio", "encoding", "escape-html", "overlapped-lists"] } +reqwest = { version = "0.11.18", default-features = false, features = ["rustls-tls", "gzip", "stream", "trust-dns"] } +serde = { version = "1.0.163", features = ["derive"] } +serde_yaml = "0.9.21" +time = { version = "0.3.21", features = ["formatting", "parsing"] } +tokio = { version = "1.28.1", features = ["full"] } +tracing = "0.1.37" +tracing-subscriber = "0.3.17" diff --git a/LICENSE b/LICENSE @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +<https://www.gnu.org/licenses/>. diff --git a/src/git/mod.rs b/src/git/mod.rs @@ -0,0 +1,116 @@ +use std::io::Write; + +use color_eyre::eyre::Result; +use git2::{Oid, Repository, Signature}; +use tracing::info; + +/// Initialize the git repository +/// +/// If the git repository already exists, open it. Otherwise, create it. +/// +/// If the git repository is created, generate the README.md file from the template. +/// +/// # Arguments +/// +/// * `git_repo_path` - The path to the git repository +/// * `data_url` - The URL to the OSM data server +/// * `changeset_url` - The URL to the OSM changeset server +/// +/// # Returns +/// +/// * `Result<Repository>` - The git repository +pub fn init_git_repository( + git_repo_path: &str, + data_url: &str, + author: &Signature, + changeset_url: &str, +) -> Result<Repository> { + // Check if the git repo already exists + if std::path::Path::new(git_repo_path).exists() { + info!("Git repository already exists at {}", git_repo_path); + // Open the git repo + let repository = Repository::open(git_repo_path)?; + + return Ok(repository); + } + + info!("Initializing git repository at {}", git_repo_path); + + // Create the git repo if it doesn't exist + let repository = Repository::init(git_repo_path)?; + + generate_readme_from_template(&repository, data_url, changeset_url)?; + + // Commit the README.md file + commit( + &repository, + vec!["README.md".to_string()], + vec![], + "Create the README.md", + author, + author, + )?; + Ok(repository) +} + +/// Generate the README.md file from the template and write it to the git repo +pub fn generate_readme_from_template( + repository: &Repository, + data_url: &str, + changeset_url: &str, +) -> Result<()> { + let template_file = include_str!("../../templates/README.md"); + + // Replace the template variables with the actual values + let template_file = template_file.replace("$server_url", data_url); + let template_file = template_file.replace("$changeset_server_url", changeset_url); + + // Get the version of this binary + let version = env!("CARGO_PKG_VERSION"); + let template_file = template_file.replace("$version", version); + + // Write the README.md file in the git repo (parent of .git directory) + let path = repository + .path() + .parent() + .expect("Git repository path is not valid"); + let readme_file_path = path.join("README.md"); + info!( + "Generating README.md file at {}", + readme_file_path + .to_str() + .expect("README.md file path is not valid") + ); + let mut readme_file = std::fs::File::create(readme_file_path)?; + readme_file.write_all(template_file.as_bytes())?; + readme_file.sync_all()?; + + info!("README.md file generated"); + + Ok(()) +} + +/// Helper for creating a git commit +pub fn commit( + repository: &Repository, + added_or_changed_files: Vec<String>, + removed_files: Vec<String>, + message: &str, + author: &Signature, + committer: &Signature, +) -> Result<Oid> { + let tree_id = { + let mut index = repository.index()?; + for file in added_or_changed_files { + index.add_path(std::path::Path::new(&file))?; + } + for file in removed_files { + index.remove_path(std::path::Path::new(&file))?; + } + index.write_tree()? + }; + let tree = repository.find_tree(tree_id)?; + + let oid = repository.commit(Some("HEAD"), author, committer, message, &tree, &[])?; + Ok(oid) +} diff --git a/src/main.rs b/src/main.rs @@ -0,0 +1,279 @@ +use std::time::Duration; + +use clap::Parser; +use color_eyre::eyre::Result; +use git2::Signature; +use tracing::{info, warn}; + +use crate::{ + git::init_git_repository, + osm::{changesets::parse_changeset, osm_data::convert_objects_to_git}, +}; + +mod git; +mod osm; + +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +struct Cli { + /// Path to the git repo to replay changesets to + #[arg(short, long, default_value = "./osm-git-repo")] + git_repo_path: String, + /// The server to get day replication files from + #[arg( + short, + long, + default_value = "https://planet.openstreetmap.org/replication/day" + )] + replication_server: String, + /// The server to get changeset data from + #[arg( + long, + default_value = "https://planet.openstreetmap.org/replication/changesets" + )] + changeset_server: String, + /// Where to write cache files + #[arg(long, default_value = "./cache")] + cache_path: String, + /// If the git repo should be removed and recreated + #[arg(short, long)] + clean: bool, + /// Where to start downloading changesets from + #[arg(long, default_value = "000/000/000")] + start_changesets: String, + /// The time to wait between downloading changesets + /// This is to avoid causing a lot of load on the OSM servers + #[arg(long, default_value = "500")] + wait_time: u64, +} + +#[tokio::main] +async fn main() -> Result<()> { + color_eyre::install()?; + tracing_subscriber::fmt::init(); + let cli = Cli::parse(); + + info!( + "Starting to replay osm changesets to git repo at {}", + cli.git_repo_path + ); + + let client = reqwest::Client::builder() + .user_agent("osm-git-replay/0.1.0") + .gzip(true) + .timeout(Duration::from_secs(60)) + .build()?; + + if cli.clean { + info!("Cleaning git repo at {}", cli.git_repo_path); + if std::path::Path::new(&cli.git_repo_path).exists() { + std::fs::remove_dir_all(&cli.git_repo_path)?; + } + } + + let author = Signature::now("osm-git-replay", "osm-git-replay@localhost")?; + + let repository = init_git_repository( + &cli.git_repo_path, + &cli.replication_server, + &author, + &cli.changeset_server, + )?; + info!("Git repository initialized"); + + // Main download loop + let mut changeset_position_top = cli.start_changesets[0..3].parse::<u64>()?; + let mut changeset_position_middle = cli.start_changesets[4..7].parse::<u64>()?; + let mut changeset_position_bottom = cli.start_changesets[8..11].parse::<u64>()?; + let mut changeset_position_middle_incremented = false; + let mut changeset_position_top_incremented = false; + + let mut changesets = Vec::new(); + loop { + // Check for cache and use it if it exists + let cache_file_path = format!( + "{}/changesets/{:03}/{:03}/{:03}.osm.gz", + cli.cache_path, + changeset_position_top, + changeset_position_middle, + changeset_position_bottom + ); + + if std::path::Path::new(&cache_file_path).exists() { + info!("Using cached changeset file at {}", cache_file_path); + let changeset_data = std::fs::read(&cache_file_path)?; + let parsed_changeset = parse_changeset(changeset_data.into())?; + info!("Changeset file parsed"); + changesets.extend(parsed_changeset); + + // Increment the changeset position + changeset_position_bottom += 1; + changeset_position_middle_incremented = false; + changeset_position_top_incremented = false; + } else { + // First we download the changeset files + let changeset_url = format!( + "{}/{:03}/{:03}/{:03}.osm.gz", + cli.changeset_server, + changeset_position_top, + changeset_position_middle, + changeset_position_bottom + ); + info!("Downloading changeset file from {}", changeset_url); + let changeset_response: reqwest::Response = client.get(&changeset_url).send().await?; + if changeset_response.status() == reqwest::StatusCode::NOT_FOUND { + warn!("Changeset file not found at {}", changeset_url); + // We've reached the end of the changesets for this bottom position. + // If we incremented top and failed again, we're done. + if changeset_position_top_incremented { + info!("Finished or failed downloading changesets"); + info!( + "Changeset position: {} {} {}", + changeset_position_top, + changeset_position_middle, + changeset_position_bottom + ); + warn!("Response body: {:?}", changeset_response.text().await?); + // TODO: We want to have an endless loop here optionally so that we can keep trying to download changesets. + break; + } + // We reset bottom to 0 and increment middle. + // We also mark middle as incremented so that we increment top on the next failure. + if !changeset_position_middle_incremented && changeset_position_bottom != 0 { + changeset_position_bottom = 0; + changeset_position_middle += 1; + changeset_position_middle_incremented = true; + changeset_position_top_incremented = false; + } else { + changeset_position_middle_incremented = false; + changeset_position_top += 1; + changeset_position_top_incremented = true; + changeset_position_middle = 0; + changeset_position_bottom = 0; + } + continue; + } + let changeset_data = changeset_response.bytes().await?; + info!("Caching changeset file to disk"); + std::fs::create_dir_all(std::path::Path::new(&cache_file_path).parent().unwrap())?; + std::fs::write(&cache_file_path, &changeset_data)?; + info!("Changeset file downloaded"); + + let parsed_changeset = parse_changeset(changeset_data)?; + info!("Changeset file parsed"); + changesets.extend(parsed_changeset); + + // Increment the changeset position + changeset_position_bottom += 1; + changeset_position_middle_incremented = false; + changeset_position_top_incremented = false; + + // Wait a few seconds before downloading the next changeset file + tokio::time::sleep(Duration::from_millis(cli.wait_time)).await; + } + } + + // Data download metadata + let mut data_position_top = 0; + let mut data_position_middle = 0; + let mut data_position_bottom = 0; + let mut data_position_middle_incremented = false; + let mut data_position_top_incremented = false; + + // Parse the changesets and convert them to git objects + loop { + // Check for cache and use it if it exists + let cache_file_path = format!( + "{}/replication/{:03}/{:03}/{:03}.osm.gz", + cli.cache_path, + changeset_position_top, + changeset_position_middle, + changeset_position_bottom + ); + + if std::path::Path::new(&cache_file_path).exists() { + info!("Using cached data file at {}", cache_file_path); + let data = std::fs::read(&cache_file_path)?; + convert_objects_to_git(&repository, &author, &changesets, &data)?; + info!("Data file parsed"); + + // Increment the data position + data_position_bottom += 1; + data_position_middle_incremented = false; + data_position_top_incremented = false; + } else { + // Download minute replication files and find the changesets that were modified in that minute + let data_url = format!( + "{}/{:03}/{:03}/{:03}.osc.gz", + cli.replication_server, + data_position_top, + data_position_middle, + data_position_bottom + ); + info!("Downloading data file from {}", data_url); + let data_response: reqwest::Response = client.get(&data_url).send().await?; + + if data_response.status() == reqwest::StatusCode::NOT_FOUND { + warn!("data file not found at {}", data_url); + // We've reached the end of the data for this bottom position. + // If we incremented top and failed again, we're done. + if data_position_top_incremented { + info!("Finished or failed downloading data"); + info!( + "Data position: {} {} {}", + data_position_top, data_position_middle, data_position_bottom + ); + warn!("Response body: {:?}", data_response.text().await?); + + // TODO: We want to have an endless loop here optionally so that we can keep trying to download changesets. + break; + } + // We reset bottom to 0 and increment middle. + // We also mark middle as incremented so that we increment top on the next failure. + if !data_position_middle_incremented && data_position_bottom != 0 { + data_position_bottom = 0; + data_position_middle += 1; + data_position_middle_incremented = true; + data_position_top_incremented = false; + } else { + data_position_middle_incremented = false; + data_position_top += 1; + data_position_top_incremented = true; + data_position_middle = 0; + data_position_bottom = 0; + } + continue; + } + + let data = data_response.bytes().await?; + info!("Caching Data file to disk"); + std::fs::create_dir_all(std::path::Path::new(&cache_file_path).parent().unwrap())?; + std::fs::write(&cache_file_path, &data)?; + info!("Data file downloaded"); + convert_objects_to_git(&repository, &author, &changesets, &data)?; + + // Increment the data position + data_position_bottom += 1; + data_position_middle_incremented = false; + data_position_top_incremented = false; + + // Wait a few seconds before downloading the next data file + tokio::time::sleep(Duration::from_millis(cli.wait_time)).await; + } + } + + info!( + "Downloaded changesets until {} {} {}", + changeset_position_top, + changeset_position_middle, + changeset_position_bottom - 1 + ); + info!( + "Downloaded data until {} {} {}", + data_position_top, + data_position_middle, + data_position_bottom - 1 + ); + + Ok(()) +} diff --git a/src/osm/changesets.rs b/src/osm/changesets.rs @@ -0,0 +1,197 @@ +use bytes::Bytes; +use color_eyre::eyre::Result; +use flate2::bufread::GzDecoder; +use quick_xml::{ + events::{BytesStart, Event}, + name::QName, + Reader, +}; +use std::{ + borrow::Cow, + collections::HashMap, + convert::Infallible, + io::{Read, Write}, +}; +use tracing::{debug, error, info, warn}; + +#[derive(Debug, Clone, PartialEq)] +pub struct Changeset { + pub id: u64, + pub created_at: String, + pub closed_at: Option<String>, + pub open: bool, + pub user: String, + pub uid: u64, + pub min_lat: Option<f64>, + pub max_lat: Option<f64>, + pub min_lon: Option<f64>, + pub max_lon: Option<f64>, + pub tags: HashMap<String, String>, +} + +impl Changeset { + fn new_from_element(reader: &mut Reader<&[u8]>, element: BytesStart) -> Result<Self> { + let changeset_attributes: HashMap<String, String> = element + .attributes() + .filter_map(|attr_result| attr_result.ok()) + .map(|attr| { + let key = reader + .decoder() + .decode(attr.key.local_name().as_ref()) + .or_else(|err| { + dbg!( + "unable to read key in DefaultSettings attribute {:?}, utf8 error {:?}", + &attr, + err + ); + Ok::<Cow<'_, str>, Infallible>(std::borrow::Cow::from("")) + }) + .unwrap() + .to_string(); + let value = attr + .decode_and_unescape_value(reader) + .or_else(|err| { + dbg!( + "unable to read key in DefaultSettings attribute {:?}, utf8 error {:?}", + &attr, + err + ); + Ok::<Cow<'_, str>, Infallible>(std::borrow::Cow::from("")) + }) + .unwrap() + .to_string(); + (key, value) + }) + .collect(); + + let mut changeset = Changeset { + id: changeset_attributes.get("id").unwrap().parse().unwrap(), + created_at: changeset_attributes.get("created_at").unwrap().to_string(), + closed_at: changeset_attributes.get("closed_at").map(|s| s.to_string()), + open: changeset_attributes.get("open").unwrap().parse().unwrap(), + user: changeset_attributes.get("user").unwrap().to_string(), + uid: changeset_attributes.get("uid").unwrap().parse().unwrap(), + min_lat: changeset_attributes + .get("min_lat") + .map(|s| s.parse().unwrap()), + max_lat: changeset_attributes + .get("max_lat") + .map(|s| s.parse().unwrap()), + min_lon: changeset_attributes + .get("min_lon") + .map(|s| s.parse().unwrap()), + max_lon: changeset_attributes + .get("max_lon") + .map(|s| s.parse().unwrap()), + tags: HashMap::new(), + }; + + let mut element_buf = Vec::new(); + loop { + let event = reader.read_event_into(&mut element_buf)?; + + if let Event::End(ref e) = event { + if e.name() == element.name() { + break; + } + } + if let Event::Start(ref e) = event { + let name = e.name(); + if name == QName(b"tag") { + let mut key = Cow::Borrowed(""); + let mut value = Cow::Borrowed(""); + + for attr_result in element.attributes() { + let a = attr_result?; + match a.key.as_ref() { + b"k" => key = a.decode_and_unescape_value(reader)?, + b"v" => value = a.decode_and_unescape_value(reader)?, + _ => (), + } + } + + changeset.tags.insert(key.to_string(), value.to_string()); + } else { + warn!("Unexpected tag: {:?}", name); + reader.read_to_end(name)?; + } + } else { + if let Event::Text(ref text) = event { + if text.borrow().starts_with(b"\n") { + continue; + } + } else if let Event::End(ref e) = event { + if e.name() == QName(b"tag") { + continue; + } + } + warn!("Unexpected event in changeset: {:?}", event); + // Write the data to file for debugging + + let mut file = std::fs::File::create("debug.xml")?; + file.write_all(&element_buf)?; + file.sync_all()?; + } + } + Ok(changeset) + } +} + +pub fn parse_changeset(changeset_data: Bytes) -> Result<Vec<Changeset>> { + // If file is empty, return an empty vector + if changeset_data.is_empty() { + return Ok(Vec::new()); + } + // Decompress the changeset file + let mut changeset_data_reader = GzDecoder::new(&changeset_data[..]); + let mut changeset_data = String::new(); + changeset_data_reader.read_to_string(&mut changeset_data)?; + debug!( + "Changeset file decompressed. Size: {}", + changeset_data.len() + ); + + // If the changeset file is empty, return an empty vector + if changeset_data.is_empty() { + return Ok(Vec::new()); + } + + let mut changeset_data = Reader::from_str(&changeset_data); + + // == Handling empty elements == + // To simply our processing code + // we want the same events for empty elements, like: + // <DefaultSettings Language="es" Greeting="HELLO"/> + // <Text/> + changeset_data.expand_empty_elements(true); + + let mut changesets = Vec::new(); + let mut buf = Vec::new(); + + // Parse the changeset file + info!("Parsing changeset file"); + loop { + let event = changeset_data.read_event_into(&mut buf)?; + match event { + Event::Start(element) => { + if let b"changeset" = element.name().as_ref() { + // TODO: What do we do in case of an error? + let changeset = + Changeset::new_from_element(&mut changeset_data, element.clone()); + match changeset { + Ok(changeset) => changesets.push(changeset), + Err(err) => { + error!( + "unable to read changeset element {:?}, utf8 error {:?}", + &element, err + ); + } + } + } + } + Event::Eof => break, // exits the loop when reaching end of file + _ => (), // There are `Event` types not considered here + } + } + Ok(changesets) +} diff --git a/src/osm/mod.rs b/src/osm/mod.rs @@ -0,0 +1,2 @@ +pub mod changesets; +pub mod osm_data; diff --git a/src/osm/osm_data.rs b/src/osm/osm_data.rs @@ -0,0 +1,927 @@ +use color_eyre::eyre::Result; +use flate2::bufread::GzDecoder; +use git2::{Repository, Signature, Time}; +use quick_xml::{ + events::{BytesStart, Event}, + name::QName, + Reader, +}; +use serde::{Deserialize, Serialize}; +use std::{ + borrow::Cow, + collections::BTreeMap, + convert::Infallible, + io::{Read, Write}, +}; +use time::{format_description::well_known::Iso8601, OffsetDateTime}; +use tracing::{debug, error, info, warn}; + +use crate::git::commit; + +use super::changesets::Changeset; + +const FILE_VERSION: &str = "0.1.0"; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Node { + /// The id of the node. Saved as the file name. + #[serde(skip)] + pub id: u64, + #[serde(skip)] + pub changeset: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub file_generator: Option<String>, + pub file_version: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub legacy_object_version: Option<String>, + pub lat: f64, + pub lon: f64, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub tags: BTreeMap<String, String>, +} +impl Node { + fn new_from_element(reader: &mut Reader<&[u8]>, element: BytesStart) -> Result<Self> { + let attributes: BTreeMap<String, String> = element + .attributes() + .filter_map(|attr_result| attr_result.ok()) + .map(|attr| { + let key = reader + .decoder() + .decode(attr.key.local_name().as_ref()) + .or_else(|err| { + dbg!( + "unable to read key in DefaultSettings attribute {:?}, utf8 error {:?}", + &attr, + err + ); + Ok::<Cow<'_, str>, Infallible>(std::borrow::Cow::from("")) + }) + .unwrap() + .to_string(); + let value = attr + .decode_and_unescape_value(reader) + .or_else(|err| { + dbg!( + "unable to read key in DefaultSettings attribute {:?}, utf8 error {:?}", + &attr, + err + ); + Ok::<Cow<'_, str>, Infallible>(std::borrow::Cow::from("")) + }) + .unwrap() + .to_string(); + (key, value) + }) + .collect(); + + let mut node = Node { + id: attributes + .get("id") + .unwrap() + .parse::<u64>() + .expect("Unable to parse node id"), + changeset: attributes + .get("changeset") + .unwrap() + .parse::<u64>() + .expect("Unable to parse node changeset"), + file_generator: attributes.get("generator").map(|s| s.to_string()), + legacy_object_version: attributes.get("version").map(|s| s.to_string()), + lat: attributes + .get("lat") + .unwrap() + .parse::<f64>() + .expect("Unable to parse node lat"), + lon: attributes + .get("lon") + .unwrap() + .parse::<f64>() + .expect("Unable to parse node lon"), + tags: BTreeMap::new(), + file_version: FILE_VERSION.to_string(), + }; + + let mut element_buf = Vec::new(); + loop { + let event = reader.read_event_into(&mut element_buf)?; + + if let Event::End(ref e) = event { + if e.name() == element.name() { + break; + } + } + + if let Event::Start(ref e) = event { + let name = e.name(); + if name == QName(b"tag") { + let mut key = Cow::Borrowed(""); + let mut value = Cow::Borrowed(""); + + for attr_result in e.attributes() { + let a = attr_result?; + match a.key.as_ref() { + b"k" => key = a.decode_and_unescape_value(reader)?, + b"v" => value = a.decode_and_unescape_value(reader)?, + _ => (), + } + } + + node.tags.insert(key.to_string(), value.to_string()); + } else { + warn!("Unexpected tag: {:?}", name); + reader.read_to_end(name)?; + } + } else { + if let Event::Text(ref text) = event { + if text.borrow().starts_with(b"\n") { + continue; + } + } else if let Event::End(ref e) = event { + if e.name() == QName(b"tag") { + continue; + } + } + warn!("Unexpected event in node: {:?}", event); + // Write the data to file for debugging + + let mut file = std::fs::File::create("debug.xml")?; + file.write_all(&element_buf)?; + file.sync_all()?; + } + } + + Ok(node) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Way { + /// The id of the node. Saved as the file name. + #[serde(skip)] + pub id: u64, + #[serde(skip)] + pub changeset: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub file_generator: Option<String>, + pub file_version: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub legacy_object_version: Option<String>, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub tags: BTreeMap<String, String>, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub nodes: Vec<u64>, +} + +impl Way { + fn new_from_element(reader: &mut Reader<&[u8]>, element: BytesStart) -> Result<Self> { + let attributes: BTreeMap<String, String> = element + .attributes() + .filter_map(|attr_result| attr_result.ok()) + .map(|attr| { + let key = reader + .decoder() + .decode(attr.key.local_name().as_ref()) + .or_else(|err| { + dbg!( + "unable to read key in DefaultSettings attribute {:?}, utf8 error {:?}", + &attr, + err + ); + Ok::<Cow<'_, str>, Infallible>(std::borrow::Cow::from("")) + }) + .unwrap() + .to_string(); + let value = attr + .decode_and_unescape_value(reader) + .or_else(|err| { + dbg!( + "unable to read key in DefaultSettings attribute {:?}, utf8 error {:?}", + &attr, + err + ); + Ok::<Cow<'_, str>, Infallible>(std::borrow::Cow::from("")) + }) + .unwrap() + .to_string(); + (key, value) + }) + .collect(); + + let mut way = Way { + id: attributes + .get("id") + .unwrap() + .parse::<u64>() + .expect("Unable to parse way id"), + changeset: attributes + .get("changeset") + .unwrap() + .parse::<u64>() + .expect("Unable to parse way changeset"), + file_generator: attributes.get("generator").map(|s| s.to_string()), + legacy_object_version: attributes.get("version").map(|s| s.to_string()), + tags: BTreeMap::new(), + nodes: Vec::new(), + file_version: FILE_VERSION.to_string(), + }; + + let mut element_buf = Vec::new(); + loop { + let event = reader.read_event_into(&mut element_buf)?; + + if let Event::End(ref e) = event { + if e.name() == element.name() { + break; + } + } + + if let Event::Start(ref e) = event { + let name = e.name(); + if name == QName(b"tag") { + let mut key = Cow::Borrowed(""); + let mut value = Cow::Borrowed(""); + + for attr_result in e.attributes() { + let a = attr_result?; + match a.key.as_ref() { + b"k" => key = a.decode_and_unescape_value(reader)?, + b"v" => value = a.decode_and_unescape_value(reader)?, + _ => (), + } + } + + way.tags.insert(key.to_string(), value.to_string()); + } else if name == QName(b"nd") { + let mut ref_id = Cow::Borrowed(""); + + for attr_result in e.attributes() { + let a = attr_result?; + if let b"ref" = a.key.as_ref() { + ref_id = a.decode_and_unescape_value(reader)?; + } + } + + way.nodes.push( + ref_id + .to_string() + .parse::<u64>() + .expect("Unable to parse way node ref"), + ); + } else { + warn!("Unexpected tag: {:?}", name); + reader.read_to_end(name)?; + } + } else { + if let Event::Text(ref text) = event { + if text.borrow().starts_with(b"\n") { + continue; + } + } else if let Event::End(ref e) = event { + if e.name() == QName(b"tag") || e.name() == QName(b"nd") { + continue; + } + } + warn!("Unexpected event way: {:?}", event); + // Write the data to file for debugging + + let mut file = std::fs::File::create("debug.xml")?; + file.write_all(&element_buf)?; + file.sync_all()?; + } + } + + Ok(way) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RelationMember { + #[serde(rename = "type")] + pub r#type: String, + #[serde(rename = "ref")] + pub ref_id: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub role: Option<String>, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Relation { + /// The id of the node. Saved as the file name. + #[serde(skip)] + pub id: u64, + #[serde(skip)] + pub changeset: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub file_generator: Option<String>, + pub file_version: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub legacy_object_version: Option<String>, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub tags: BTreeMap<String, String>, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub member: Vec<RelationMember>, +} + +impl Relation { + fn new_from_element(reader: &mut Reader<&[u8]>, element: BytesStart) -> Result<Self> { + let attributes: BTreeMap<String, String> = element + .attributes() + .filter_map(|attr_result| attr_result.ok()) + .map(|attr| { + let key = reader + .decoder() + .decode(attr.key.local_name().as_ref()) + .or_else(|err| { + dbg!( + "unable to read key in DefaultSettings attribute {:?}, utf8 error {:?}", + &attr, + err + ); + Ok::<Cow<'_, str>, Infallible>(std::borrow::Cow::from("")) + }) + .unwrap() + .to_string(); + let value = attr + .decode_and_unescape_value(reader) + .or_else(|err| { + dbg!( + "unable to read key in DefaultSettings attribute {:?}, utf8 error {:?}", + &attr, + err + ); + Ok::<Cow<'_, str>, Infallible>(std::borrow::Cow::from("")) + }) + .unwrap() + .to_string(); + (key, value) + }) + .collect(); + + let mut relation = Relation { + id: attributes + .get("id") + .unwrap() + .parse::<u64>() + .expect("Unable to parse way id"), + changeset: attributes + .get("changeset") + .unwrap() + .parse::<u64>() + .expect("Unable to parse way changeset"), + file_generator: attributes.get("generator").map(|s| s.to_string()), + legacy_object_version: attributes.get("version").map(|s| s.to_string()), + tags: BTreeMap::new(), + member: Vec::new(), + file_version: FILE_VERSION.to_string(), + }; + + let mut element_buf = Vec::new(); + loop { + let event = reader.read_event_into(&mut element_buf)?; + + if let Event::End(ref e) = event { + if e.name() == element.name() { + break; + } + } + + if let Event::Start(ref e) = event { + let name = e.name(); + if name == QName(b"tag") { + let mut key = Cow::Borrowed(""); + let mut value = Cow::Borrowed(""); + + for attr_result in e.attributes() { + let a = attr_result?; + match a.key.as_ref() { + b"k" => key = a.decode_and_unescape_value(reader)?, + b"v" => value = a.decode_and_unescape_value(reader)?, + _ => (), + } + } + + relation.tags.insert(key.to_string(), value.to_string()); + } else if name == QName(b"member") { + let mut ref_id = Cow::Borrowed(""); + let mut r#type = Cow::Borrowed(""); + let mut role = Cow::Borrowed(""); + + for attr_result in e.attributes() { + let a = attr_result?; + match a.key.as_ref() { + b"ref" => ref_id = a.decode_and_unescape_value(reader)?, + b"type" => r#type = a.decode_and_unescape_value(reader)?, + b"role" => role = a.decode_and_unescape_value(reader)?, + _ => (), + } + } + + let normalized_role = if role.is_empty() { + None + } else { + Some(role.to_string()) + }; + + relation.member.push(RelationMember { + r#type: r#type.to_string(), + ref_id: ref_id + .to_string() + .parse::<u64>() + .expect("Unable to parse relation member ref"), + role: normalized_role, + }); + } else { + warn!("Unexpected tag: {:?}", name); + reader.read_to_end(name)?; + } + } else { + if let Event::Text(ref text) = event { + if text.borrow().starts_with(b"\n") { + continue; + } + } else if let Event::End(ref e) = event { + if e.name() == QName(b"tag") || e.name() == QName(b"member") { + continue; + } + } + warn!("Unexpected event in Relation: {:?}", event); + // Write the data to file for debugging + + let mut file = std::fs::File::create("debug.xml")?; + file.write_all(&element_buf)?; + file.sync_all()?; + } + } + + Ok(relation) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum OSMObject { + Node(Node), + Way(Way), + Relation(Relation), +} + +pub fn convert_objects_to_git( + repository: &Repository, + committer: &Signature, + changesets: &[Changeset], + data: &[u8], +) -> Result<()> { + // If the file is empty we skip it + if data.is_empty() { + return Ok(()); + } + + // Decompress the changeset file + let mut data_reader = GzDecoder::new(data); + let mut file_data = String::new(); + data_reader.read_to_string(&mut file_data)?; + debug!("Data file decompressed. Size: {}", file_data.len()); + + // If the file is empty we skip it + if file_data.is_empty() { + return Ok(()); + } + + info!("Parsing data file"); + + let mut data = Reader::from_str(&file_data); + + // == Handling empty elements == + // To simply our processing code + // we want the same events for empty elements, like: + // <DefaultSettings Language="es" Greeting="HELLO"/> + // <Text/> + data.expand_empty_elements(true); + + let mut buf = Vec::new(); + let mut created_or_modified_objects_for_changeset = BTreeMap::new(); + let mut deleted_objects_for_changeset = BTreeMap::new(); + + loop { + let event: Event = data.read_event_into(&mut buf)?; + match event { + Event::Start(element) => match element.name().as_ref() { + b"create" => { + // TODO: What do we do in case of an error? + + let mut created_objects = Vec::new(); + + let mut element_buf = Vec::new(); + loop { + let event = data.read_event_into(&mut element_buf)?; + + if let Event::End(ref e) = event { + if e.name() == element.name() { + break; + } + } + + if let Event::Start(ref e) = event { + let name = e.name(); + if name == QName(b"node") { + let node = Node::new_from_element(&mut data, e.clone()); + match node { + Ok(node) => created_objects.push(OSMObject::Node(node)), + Err(err) => { + error!( + "unable to read node element {:?}, utf8 error {:?}", + &e, err + ); + } + } + } else if name == QName(b"way") { + let way = Way::new_from_element(&mut data, e.clone()); + match way { + Ok(way) => created_objects.push(OSMObject::Way(way)), + Err(err) => { + error!( + "unable to read way element {:?}, utf8 error {:?}", + &e, err + ); + } + } + } else if name == QName(b"relation") { + let relation = Relation::new_from_element(&mut data, e.clone()); + match relation { + Ok(relation) => { + created_objects.push(OSMObject::Relation(relation)) + } + Err(err) => { + error!( + "unable to read relation element {:?}, utf8 error {:?}", + &e, err + ); + } + } + } else { + warn!("Unexpected tag: {:?}", name); + data.read_to_end(name)?; + } + } else { + if let Event::Text(ref text) = event { + if text.borrow().starts_with(b"\n") { + continue; + } + } + warn!("Unexpected event in create: {:?}", event); + // Write the data to file for debugging + + let mut file = std::fs::File::create("debug.xml")?; + file.write_all(file_data.as_bytes())?; + file.sync_all()?; + } + } + + // write the objects to the git repo as yaml files + let repository_folder = repository.path().parent().unwrap(); + for object in created_objects { + let object_file_name = match object { + OSMObject::Node(ref node) => format!("{}.yaml", node.id), + OSMObject::Way(ref way) => format!("{}.yaml", way.id), + OSMObject::Relation(ref relation) => format!("{}.yaml", relation.id), + }; + let object_file_path = repository_folder.join(object_file_name); + let object_file = std::fs::File::create(object_file_path)?; + serde_yaml::to_writer(object_file, &object)?; + + // Add the object to the list of created objects for the changeset based on the changeset id + let changeset = match object { + OSMObject::Node(ref node) => node.changeset, + OSMObject::Way(ref way) => way.changeset, + OSMObject::Relation(ref relation) => relation.changeset, + }; + created_or_modified_objects_for_changeset + .entry(changeset) + .or_insert_with(Vec::new) + .push(object); + } + } + b"modify" => { + // TODO: What do we do in case of an error? + + let mut deleted_objects = Vec::new(); + + let mut element_buf = Vec::new(); + loop { + let event = data.read_event_into(&mut element_buf)?; + + if let Event::End(ref e) = event { + if e.name() == element.name() { + break; + } + } + + if let Event::Start(ref e) = event { + let name = e.name(); + if name == QName(b"node") { + let node = Node::new_from_element(&mut data, e.clone()); + match node { + Ok(node) => deleted_objects.push(OSMObject::Node(node)), + Err(err) => { + error!( + "unable to read node element {:?}, utf8 error {:?}", + &e, err + ); + } + } + } else if name == QName(b"way") { + let way = Way::new_from_element(&mut data, e.clone()); + match way { + Ok(way) => deleted_objects.push(OSMObject::Way(way)), + Err(err) => { + error!( + "unable to read way element {:?}, utf8 error {:?}", + &e, err + ); + } + } + } else if name == QName(b"relation") { + let relation = Relation::new_from_element(&mut data, e.clone()); + match relation { + Ok(relation) => { + deleted_objects.push(OSMObject::Relation(relation)) + } + Err(err) => { + error!( + "unable to read relation element {:?}, utf8 error {:?}", + &e, err + ); + } + } + } else { + warn!("Unexpected tag: {:?}", name); + data.read_to_end(name)?; + } + } else { + if let Event::Text(ref text) = event { + if text.borrow().starts_with(b"\n") { + continue; + } + } + warn!("Unexpected event in create: {:?}", event); + // Write the data to file for debugging + + let mut file = std::fs::File::create("debug.xml")?; + file.write_all(file_data.as_bytes())?; + file.sync_all()?; + } + } + + // write the objects to the git repo as yaml files + let repository_folder = repository.path().parent().unwrap(); + for object in deleted_objects { + let object_file_name = match object { + OSMObject::Node(ref node) => format!("{}.yaml", node.id), + OSMObject::Way(ref way) => format!("{}.yaml", way.id), + OSMObject::Relation(ref relation) => format!("{}.yaml", relation.id), + }; + let object_file_path = repository_folder.join(object_file_name); + + // Change the file according to the changeset + let mut object_file = std::fs::File::open(object_file_path)?; + let mut file_object: OSMObject = serde_yaml::from_reader(&mut object_file)?; + + match object { + OSMObject::Node(ref node) => { + if let OSMObject::Node(ref mut file_node) = file_object { + file_node.changeset = node.changeset; + file_node.file_generator = node.file_generator.clone(); + file_node.file_version = node.file_version.clone(); + file_node.legacy_object_version = + node.legacy_object_version.clone(); + file_node.lat = node.lat; + file_node.lon = node.lon; + file_node.tags = node.tags.clone(); + } + } + OSMObject::Way(ref way) => { + if let OSMObject::Way(ref mut file_way) = file_object { + file_way.changeset = way.changeset; + file_way.file_generator = way.file_generator.clone(); + file_way.file_version = way.file_version.clone(); + file_way.legacy_object_version = + way.legacy_object_version.clone(); + file_way.tags = way.tags.clone(); + file_way.nodes = way.nodes.clone(); + } + } + OSMObject::Relation(ref relation) => { + if let OSMObject::Relation(ref mut file_relation) = file_object { + file_relation.changeset = relation.changeset; + file_relation.file_generator = relation.file_generator.clone(); + file_relation.file_version = relation.file_version.clone(); + file_relation.legacy_object_version = + relation.legacy_object_version.clone(); + file_relation.tags = relation.tags.clone(); + file_relation.member = relation.member.clone(); + } + } + } + // Add the object to the list of created objects for the changeset based on the changeset id + let changeset = match object { + OSMObject::Node(ref node) => node.changeset, + OSMObject::Way(ref way) => way.changeset, + OSMObject::Relation(ref relation) => relation.changeset, + }; + created_or_modified_objects_for_changeset + .entry(changeset) + .or_insert_with(Vec::new) + .push(object); + } + } + b"delete" => { + // TODO: What do we do in case of an error? + + let mut deleted_objects = Vec::new(); + + let mut element_buf = Vec::new(); + loop { + let event = data.read_event_into(&mut element_buf)?; + + if let Event::End(ref e) = event { + if e.name() == element.name() { + break; + } + } + + if let Event::Start(ref e) = event { + let name = e.name(); + if name == QName(b"node") { + let node = Node::new_from_element(&mut data, e.clone()); + match node { + Ok(node) => deleted_objects.push(OSMObject::Node(node)), + Err(err) => { + error!( + "unable to read node element {:?}, utf8 error {:?}", + &e, err + ); + } + } + } else if name == QName(b"way") { + let way = Way::new_from_element(&mut data, e.clone()); + match way { + Ok(way) => deleted_objects.push(OSMObject::Way(way)), + Err(err) => { + error!( + "unable to read way element {:?}, utf8 error {:?}", + &e, err + ); + } + } + } else if name == QName(b"relation") { + let relation = Relation::new_from_element(&mut data, e.clone()); + match relation { + Ok(relation) => { + deleted_objects.push(OSMObject::Relation(relation)) + } + Err(err) => { + error!( + "unable to read relation element {:?}, utf8 error {:?}", + &e, err + ); + } + } + } else { + warn!("Unexpected tag: {:?}", name); + data.read_to_end(name)?; + } + } else { + if let Event::Text(ref text) = event { + if text.borrow().starts_with(b"\n") { + continue; + } + } + warn!("Unexpected event in create: {:?}", event); + // Write the data to file for debugging + + let mut file = std::fs::File::create("debug.xml")?; + file.write_all(file_data.as_bytes())?; + file.sync_all()?; + } + } + + // write the objects to the git repo as yaml files + let repository_folder = repository.path().parent().unwrap(); + for object in deleted_objects { + let object_file_name = match object { + OSMObject::Node(ref node) => format!("{}.yaml", node.id), + OSMObject::Way(ref way) => format!("{}.yaml", way.id), + OSMObject::Relation(ref relation) => format!("{}.yaml", relation.id), + }; + let object_file_path = repository_folder.join(object_file_name); + + // Delete the file + std::fs::remove_file(object_file_path)?; + + // Add the object to the list of created objects for the changeset based on the changeset id + let changeset = match object { + OSMObject::Node(ref node) => node.changeset, + OSMObject::Way(ref way) => way.changeset, + OSMObject::Relation(ref relation) => relation.changeset, + }; + deleted_objects_for_changeset + .entry(changeset) + .or_insert_with(Vec::new) + .push(object); + } + } + _ => (), + }, + Event::Eof => break, // exits the loop when reaching end of file + _ => (), // There are `Event` types not considered here + } + } + + // For all the objects changed apply the changesets as commits + // Get changeset list from BTreeMaps + let changeset_list: Vec<&u64> = created_or_modified_objects_for_changeset + .keys() + .chain(deleted_objects_for_changeset.keys()) + .collect(); + + for changeset in changeset_list { + // Construct the commit and apply it + let changeset = changesets + .iter() + .find(|c| c.id == *changeset) + .expect("Unable to find changeset"); + + // Get comment tag if it exists and trim it + let comment = changeset + .tags + .get("comment") + .map(|s| s.trim()) + .unwrap_or(""); + + // Parse changeset time (ISO 8601) to git time (seconds since epoch) with offset 0 (UTC) using `time` + let changeset_time = changeset + .closed_at + .clone() + .unwrap_or(changeset.created_at.clone()); + let commit_time = + OffsetDateTime::parse(changeset_time.as_str(), &Iso8601::DEFAULT)?.unix_timestamp(); + + let author = git2::Signature::new( + &changeset.user, + &format!("{}@osm", changeset.user), + &Time::new(commit_time, 0), + ) + .expect("Unable to create author signature"); + + let repository_folder = repository.path().parent().unwrap(); + + let added_or_changed_files = created_or_modified_objects_for_changeset + .get(&changeset.id) + .unwrap_or(&Vec::new()) + .iter() + .map(|object| match object { + OSMObject::Node(ref node) => repository_folder.join(format!("{}.yaml", node.id)), + OSMObject::Way(ref way) => repository_folder.join(format!("{}.yaml", way.id)), + OSMObject::Relation(ref relation) => { + repository_folder.join(format!("{}.yaml", relation.id)) + } + }) + .map(|path| path.to_string_lossy().to_string()) + .collect::<Vec<String>>(); + + let removed_files = deleted_objects_for_changeset + .get(&changeset.id) + .unwrap_or(&Vec::new()) + .iter() + .map(|object| match object { + OSMObject::Node(ref node) => repository_folder.join(format!("{}.yaml", node.id)), + OSMObject::Way(ref way) => repository_folder.join(format!("{}.yaml", way.id)), + OSMObject::Relation(ref relation) => { + repository_folder.join(format!("{}.yaml", relation.id)) + } + }) + .map(|path| path.to_string_lossy().to_string()) + .collect::<Vec<String>>(); + + let oid = commit( + repository, + added_or_changed_files, + removed_files, + comment, + &author, + committer, + )?; + + // Convert tags to "Key: Value" strings separated by newlines for the note + let note = changeset + .tags + .iter() + .map(|(key, value)| format!("{}: {}", key, value)) + .collect::<Vec<String>>() + .join("\n"); + + repository.note(&author, committer, None, oid, &note, false)?; + } + + Ok(()) +} diff --git a/templates/README.md b/templates/README.md @@ -0,0 +1,11 @@ +# OSM-Mirror + +This is a mirror of the OpenStreetMap database. It is rebuild from the +<$server_url> server data files and the <$changeset_server_url> +changeset files. + +This mirror is autogenerated currently. + +It is based on the idea from <https://blog.andygol.co.ua/en/2023/05/07/osm-2-0-api-using-git/> + +The Script used to build this is osm-git version $version.