commit 0a07140755363c7c3bbb9bb8ca4576c39dba5a32
parent 5db1e3e9b1e41054c1e07b4fea1c8ff6ea7cd9bf
Author: mauricefisher64 <92736594+mauricefisher64@users.noreply.github.com>
Date: Wed, 24 May 2023 16:39:52 -0400
Bmff v2 (#251)
* Check for more than one manifest store
* Fix grabbing wrong UUID
* Updated to fix playback issue
* Add comment
* Initial code
* BMFF 1.1 hash support
* fix missing file
* Test wasm fix
* Slightly more readable check
* Better variable name
* Fix typos in comments
* Update to incorporate comments
* Don't enable /mdat exclusions yet
* Initial Merkle support
* More Merkle support
* Streaming BMFF hash support
* Merge fixes
* Add fragment API to manifest_store
* remove print statements
* Fix broken sample hash
* Remove test code
* Add tree_dump
* Rename Byte to Bytes
* Fix build issue after merge
* Format fixes
* Format fixes
* Another format fix
* Added description for hashing since it is complex
* Code cleanup to move Merkle validation to the MerkleMap impl
Bug fix in Merkle proof validation
Support for validating fragmented content in a single file using Merkle hashing.
* missing changes
* remove dead code
* A little cleanup
* Typo fixes
* Slight cleanup
* Fix crasher in treedump for some assets
Diffstat:
13 files changed, 1405 insertions(+), 265 deletions(-)
diff --git a/sdk/Cargo.toml b/sdk/Cargo.toml
@@ -69,6 +69,7 @@ log = "0.4.8"
lazy_static = "1.4.0"
multibase = "0.9.0"
multihash = "0.11.4"
+mp4 = "0.13.0"
png_pong = "0.8.2"
range-set = "0.0.9"
ring = "0.16.20"
diff --git a/sdk/src/assertions/bmff_hash.rs b/sdk/src/assertions/bmff_hash.rs
@@ -12,21 +12,36 @@
// each license.
use std::{
- fs,
- io::Cursor,
+ cmp,
+ collections::{hash_map::Entry::Vacant, HashMap},
+ fmt, fs,
+ io::{BufReader, Cursor, SeekFrom},
+ ops::Deref,
path::{Path, PathBuf},
};
-use serde::{Deserialize, Serialize};
+use mp4::*;
+use serde::{
+ de::{SeqAccess, Visitor},
+ ser::SerializeSeq,
+ Deserialize, Deserializer, Serialize, Serializer,
+};
use serde_bytes::ByteBuf;
+use sha2::{Digest, Sha256, Sha384, Sha512};
use crate::{
assertion::{Assertion, AssertionBase, AssertionCbor},
assertions::labels,
- asset_handlers::bmff_io::bmff_to_jumbf_exclusions,
+ asset_handlers::bmff_io::{bmff_to_jumbf_exclusions, read_bmff_c2pa_boxes, BoxInfoLite},
+ asset_io::CAIRead,
cbor_types::UriT,
- error::Result,
- utils::hash_utils::{hash_asset_by_alg, verify_asset_by_alg, verify_by_alg},
+ utils::{
+ hash_utils::{
+ concat_and_hash, hash_asset_by_alg, hash_stream_by_alg, vec_compare,
+ verify_stream_by_alg, HashRange, Hasher,
+ },
+ merkle::C2PAMerkleTree,
+ },
Error,
};
@@ -57,6 +72,63 @@ impl ExclusionsMap {
}
}
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct VecByteBuf(Vec<ByteBuf>);
+
+impl Deref for VecByteBuf {
+ type Target = Vec<ByteBuf>;
+
+ fn deref(&self) -> &Vec<ByteBuf> {
+ &self.0
+ }
+}
+
+impl Serialize for VecByteBuf {
+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
+ where
+ S: Serializer,
+ {
+ let mut seq = serializer.serialize_seq(Some(self.0.len()))?;
+ for e in &self.0 {
+ seq.serialize_element(e)?;
+ }
+ seq.end()
+ }
+}
+
+struct VecByteBufVisitor;
+
+impl<'de> Visitor<'de> for VecByteBufVisitor {
+ type Value = VecByteBuf;
+
+ fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+ formatter.write_str("Vec<ByteBuf>")
+ }
+
+ fn visit_seq<V>(self, mut visitor: V) -> std::result::Result<Self::Value, V::Error>
+ where
+ V: SeqAccess<'de>,
+ {
+ let len = cmp::min(visitor.size_hint().unwrap_or(0), 4096);
+ let mut byte_bufs: Vec<ByteBuf> = Vec::with_capacity(len);
+
+ while let Some(b) = visitor.next_element()? {
+ byte_bufs.push(b);
+ }
+
+ Ok(VecByteBuf(byte_bufs))
+ }
+}
+
+impl<'de> Deserialize<'de> for VecByteBuf {
+ fn deserialize<D>(deserializer: D) -> std::result::Result<VecByteBuf, D::Error>
+ where
+ D: Deserializer<'de>,
+ {
+ deserializer.deserialize_seq(VecByteBufVisitor {})
+ }
+}
+
#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)]
pub struct MerkleMap {
#[serde(rename = "uniqueId")]
@@ -70,10 +142,82 @@ pub struct MerkleMap {
#[serde(skip_serializing_if = "Option::is_none")]
pub alg: Option<String>,
- #[serde(rename = "initHash")]
- pub init_hash: Vec<u8>,
+ #[serde(rename = "initHash", skip_serializing_if = "Option::is_none")]
+ pub init_hash: Option<ByteBuf>,
+
+ pub hashes: VecByteBuf,
+}
+
+impl MerkleMap {
+ pub fn hash_check(&self, indx: u32, merkle_hash: &[u8]) -> bool {
+ if let Some(h) = self.hashes.get(indx as usize) {
+ vec_compare(h, merkle_hash)
+ } else {
+ false
+ }
+ }
+
+ pub fn check_merkle_tree(
+ &self,
+ alg: &str,
+ hash: &[u8],
+ location: u32,
+ proof: &Option<VecByteBuf>,
+ ) -> bool {
+ let mut index = location;
+ let mut hash = hash.to_vec();
+
+ if let Some(hashes) = proof {
+ let layers = C2PAMerkleTree::to_layout(self.count as usize);
+
+ // playback proof
+ let mut proof_index = 0;
+ for layer in layers {
+ let is_right = index % 2 == 1;
+
+ if layer == self.hashes.len() {
+ break;
+ }
+
+ if is_right {
+ if index - 1 < layer as u32 {
+ // make sure proof structure is valid
+ if let Some(proof_hash) = hashes.get(proof_index) {
+ hash = concat_and_hash(alg, proof_hash, Some(&hash));
+ proof_index += 1;
+ } else {
+ return false;
+ }
+ }
+ } else if index + 1 < layer as u32 {
+ // make sure proof structure is valid
+ if let Some(proof_hash) = hashes.get(proof_index) {
+ hash = concat_and_hash(alg, &hash, Some(proof_hash));
+ proof_index += 1;
+ } else {
+ return false;
+ }
+ }
+
+ index /= 2;
+ }
+ }
- pub hashes: Vec<ByteBuf>,
+ self.hash_check(index, &hash)
+ }
+}
+
+#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)]
+pub struct BmffMerkleMap {
+ #[serde(rename = "uniqueId")]
+ pub unique_id: u32,
+
+ #[serde(rename = "localId")]
+ pub local_id: u32,
+
+ pub location: u32,
+
+ pub hashes: Option<VecByteBuf>,
}
#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)]
@@ -97,8 +241,8 @@ pub struct BmffHash {
#[serde(skip_serializing_if = "Option::is_none")]
alg: Option<String>,
- #[serde(with = "serde_bytes")]
- hash: Vec<u8>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ hash: Option<ByteBuf>,
#[serde(skip_serializing_if = "Option::is_none")]
merkle: Option<Vec<MerkleMap>>,
@@ -106,8 +250,8 @@ pub struct BmffHash {
#[serde(skip_serializing_if = "Option::is_none")]
name: Option<String>,
- #[serde(skip_serializing_if = "Option::is_none")]
- url: Option<UriT>,
+ #[serde(skip_serializing)]
+ url: Option<UriT>, // deprecated in V2 and not to be used
#[serde(skip)]
pub path: PathBuf,
@@ -126,7 +270,7 @@ impl BmffHash {
BmffHash {
exclusions: Vec::new(),
alg: Some(alg.to_string()),
- hash: Vec::new(),
+ hash: None,
merkle: None,
name: Some(name.to_string()),
url,
@@ -147,12 +291,16 @@ impl BmffHash {
self.alg.as_ref()
}
- pub fn hash(&self) -> &[u8] {
- self.hash.as_ref()
+ pub fn hash(&self) -> Option<&Vec<u8>> {
+ self.hash.as_deref()
+ }
+
+ pub fn merkle(&self) -> Option<&Vec<MerkleMap>> {
+ self.merkle.as_ref()
}
pub fn set_hash(&mut self, hash: Vec<u8>) {
- self.hash = hash;
+ self.hash = Some(ByteBuf::from(hash));
}
pub fn name(&self) -> Option<&String> {
@@ -181,22 +329,22 @@ impl BmffHash {
}
/// Generate the hash value for the asset using the range from the BmffHash.
- pub fn gen_hash(&mut self, asset_path: &Path) -> Result<()> {
- self.hash = self.hash_from_asset(asset_path)?;
+ pub fn gen_hash(&mut self, asset_path: &Path) -> crate::error::Result<()> {
+ self.hash = Some(ByteBuf::from(self.hash_from_asset(asset_path)?));
self.path = PathBuf::from(asset_path);
Ok(())
}
/// Generate the hash again.
- pub fn regen_hash(&mut self) -> Result<()> {
+ pub fn regen_hash(&mut self) -> crate::error::Result<()> {
let p = self.path.clone();
- self.hash = self.hash_from_asset(p.as_path())?;
+ self.hash = Some(ByteBuf::from(self.hash_from_asset(p.as_path())?));
Ok(())
}
/// Generate the asset hash from a file asset using the constructed
/// start and length values.
- fn hash_from_asset(&mut self, asset_path: &Path) -> Result<Vec<u8>> {
+ fn hash_from_asset(&mut self, asset_path: &Path) -> crate::error::Result<Vec<u8>> {
if self.is_remote_hash() {
return Err(Error::BadParam(
"asset hash is remote, not yet supported".to_owned(),
@@ -224,45 +372,464 @@ impl BmffHash {
}
}
- pub fn verify_in_memory_hash(&self, data: &[u8], alg: Option<String>) -> Result<()> {
+ pub fn verify_in_memory_hash(
+ &self,
+ data: &[u8],
+ alg: Option<&str>,
+ ) -> crate::error::Result<()> {
+ let mut reader = Cursor::new(data);
+
+ self.verify_stream(&mut reader, alg)
+ }
+
+ // The BMFFMerklMaps are stored contiguous in the file. Break this Vec into groups based on
+ // the MerkleMap it matches.
+ fn split_bmff_merkle_map(
+ &self,
+ bmff_merkle_map: Vec<BmffMerkleMap>,
+ ) -> crate::Result<HashMap<u32, Vec<BmffMerkleMap>>> {
+ let mut current = bmff_merkle_map;
+ let mut output = HashMap::new();
+ if let Some(mm) = self.merkle() {
+ for m in mm {
+ let rest = current.split_off(m.count as usize);
+
+ if current.len() == m.count as usize {
+ output.insert(m.local_id, current.to_owned());
+ } else {
+ return Err(Error::HashMismatch("MerkleMap count incorrect".to_string()));
+ }
+ current = rest;
+ }
+ } else {
+ output.insert(0, current);
+ }
+ Ok(output)
+ }
+
+ // Breaks box runs at fragment boundaries (moof boxes)
+ fn split_fragment_boxes(boxes: &[BoxInfoLite]) -> Vec<Vec<BoxInfoLite>> {
+ let mut moof_list = Vec::new();
+
+ // start from 1st moof
+ if let Some(pos) = boxes.iter().position(|b| b.path == "moof") {
+ let mut box_list = vec![boxes[pos].clone()];
+
+ for b in boxes[pos + 1..].iter() {
+ if b.path == "moof" {
+ moof_list.push(box_list); // save box list
+ box_list = Vec::new(); // start new box list
+ }
+ box_list.push(b.clone());
+ }
+ moof_list.push(box_list); // save last list
+ }
+ moof_list
+ }
+
+ pub fn verify_hash(&self, asset_path: &Path, alg: Option<&str>) -> crate::error::Result<()> {
+ let mut data = fs::File::open(asset_path)?;
+ self.verify_stream(&mut data, alg)
+ }
+
+ /* Verifies BMFF hashes from a single file asset. The following variants are handled
+ A single BMFF asset with only a file hash
+ A single BMMF asset with Merkle tree hash
+ Timed media (Merkle hashes over track chunks)
+ Untimed media (Merkle hashes over iloc locations)
+ A single BMFF asset containing all fragments (Merkle hashes over moof ranges).
+ */
+ pub fn verify_stream(
+ &self,
+ reader: &mut dyn CAIRead,
+ alg: Option<&str>,
+ ) -> crate::error::Result<()> {
+ if self.is_remote_hash() {
+ return Err(Error::BadParam(
+ "asset hash is remote, not yet supported".to_owned(),
+ ));
+ }
+
+ reader.rewind()?;
+ let size = stream_len(reader)?;
+
let curr_alg = match &self.alg {
Some(a) => a.clone(),
None => match alg {
- Some(a) => a,
+ Some(a) => a.to_owned(),
None => "sha256".to_string(),
},
};
- let bmff_exclusions = &self.exclusions;
-
- let mut data_reader = Cursor::new(data);
-
// convert BMFF exclusion map to flat exclusion list
- let exclusions =
- bmff_to_jumbf_exclusions(&mut data_reader, bmff_exclusions, self.bmff_version > 1)?;
+ let exclusions = bmff_to_jumbf_exclusions(reader, &self.exclusions, self.bmff_version > 1)?;
+
+ // handle file level hashing
+ if let Some(hash) = self.hash() {
+ if !verify_stream_by_alg(&curr_alg, hash, reader, Some(exclusions.clone()), true) {
+ return Err(Error::HashMismatch(
+ "BMFF file level hash mismatch".to_string(),
+ ));
+ }
+ }
- if verify_by_alg(&curr_alg, &self.hash, data, Some(exclusions)) {
- Ok(())
- } else {
- Err(Error::HashMismatch("Hashes do not match".to_owned()))
+ // merkle hashed BMFF
+ if let Some(mm_vec) = self.merkle() {
+ // get merkle boxes from asset
+ let c2pa_boxes = read_bmff_c2pa_boxes(reader)?;
+ let bmff_merkle = c2pa_boxes.bmff_merkle;
+ let box_infos = c2pa_boxes.box_infos;
+
+ let first_moof = box_infos.iter().find(|b| b.path == "moof");
+ let is_fragmented = first_moof.is_some();
+
+ // check initialization segments (must do here in separate loop since MP4 will consume the reader)
+ for mm in mm_vec {
+ let alg = match &mm.alg {
+ Some(a) => a,
+ None => self
+ .alg()
+ .ok_or(Error::HashMismatch("no algorithm found".to_string()))?,
+ };
+
+ if let Some(init_hash) = &mm.init_hash {
+ if let Some(moof_box) = first_moof {
+ // add the moof to end exclusion
+ let moof_exclusion = HashRange::new(
+ moof_box.offset as usize,
+ (size - moof_box.offset) as usize,
+ );
+
+ let mut mm_exclusions = exclusions.clone();
+ mm_exclusions.push(moof_exclusion);
+
+ if !verify_stream_by_alg(alg, init_hash, reader, Some(mm_exclusions), true)
+ {
+ return Err(Error::HashMismatch(
+ "BMFF file level hash mismatch".to_string(),
+ ));
+ }
+ } else {
+ return Err(Error::HashMismatch(
+ "BMFF inithash must not be present for non-fragmented media".to_owned(),
+ ));
+ }
+ }
+ }
+
+ // is this a fragmented BMFF
+ if is_fragmented {
+ for mm in mm_vec {
+ let alg = match &mm.alg {
+ Some(a) => a,
+ None => self
+ .alg()
+ .ok_or(Error::HashMismatch("no algorithm found".to_string()))?,
+ };
+
+ let moof_chunks = BmffHash::split_fragment_boxes(&box_infos);
+
+ // make sure there is a 1-1 mapping of moof chunks and Merkle values
+ if moof_chunks.len() != mm.count as usize
+ || bmff_merkle.len() != mm.count as usize
+ {
+ return Err(Error::HashMismatch(
+ "Incorrect number of fragments hashes".to_owned(),
+ ));
+ }
+
+ // build Merkle tree for the moof chucks minus the excluded ranges
+ for (index, boxes) in moof_chunks.iter().enumerate() {
+ // include just the range of this chunk so exclude boxes before and after
+ let mut curr_exclusions = exclusions.clone();
+
+ // before box exclusion starts at beginning of file until the start of this chunk
+ let before_box_start = 0;
+ let before_box_len = match boxes.first() {
+ Some(first) => first.offset as usize,
+ None => 0,
+ };
+ let before_box_exclusion = HashRange::new(before_box_start, before_box_len);
+ curr_exclusions.push(before_box_exclusion);
+
+ // after box exclusion continues to the end of the file
+ let after_box_start = match boxes.last() {
+ Some(last) => last.offset + last.size,
+ None => 0,
+ };
+ let after_box_len = size - after_box_start;
+ let after_box_exclusion =
+ HashRange::new(after_box_start as usize, after_box_len as usize);
+ curr_exclusions.push(after_box_exclusion);
+
+ // hash the specified range
+ let hash = hash_stream_by_alg(alg, reader, Some(curr_exclusions), true)?;
+
+ let bmff_mm = &bmff_merkle[index];
+
+ // check MerkleMap for the hash
+ if !mm.check_merkle_tree(alg, &hash, bmff_mm.location, &bmff_mm.hashes) {
+ return Err(Error::HashMismatch("Fragment not valid".to_string()));
+ }
+ }
+ }
+ return Ok(());
+ } else if box_infos.iter().any(|b| b.path == "moov") {
+ // timed media case
+
+ let track_to_bmff_merkle_map = if bmff_merkle.is_empty() {
+ HashMap::new()
+ } else {
+ self.split_bmff_merkle_map(bmff_merkle)?
+ };
+
+ reader.rewind()?;
+ let buf_reader = BufReader::new(reader);
+ let mut mp4 = mp4::Mp4Reader::read_header(buf_reader, size)
+ .map_err(|_e| Error::InvalidAsset("Could not parse BMFF".to_string()))?;
+ let track_count = mp4.tracks().len();
+
+ for mm in mm_vec {
+ let alg = match &mm.alg {
+ Some(a) => a,
+ None => self
+ .alg()
+ .ok_or(Error::HashMismatch("no algorithm found".to_string()))?,
+ };
+
+ if track_count > 0 {
+ // timed media case
+ let track = {
+ // clone so we can borrow later
+ let tt = mp4.tracks().get(&mm.local_id).ok_or(Error::HashMismatch(
+ "Merkle location not found".to_owned(),
+ ))?;
+
+ Mp4Track {
+ trak: tt.trak.clone(),
+ trafs: tt.trafs.clone(),
+ default_sample_duration: tt.default_sample_duration,
+ }
+ };
+
+ let sample_cnt = mp4.sample_count(mm.local_id).map_err(|_e| {
+ Error::InvalidAsset("Could not parse BMFF track sample".to_string())
+ })?;
+
+ if sample_cnt == 0 {
+ return Err(Error::InvalidAsset("No samples".to_string()));
+ }
+
+ let track_id = track.track_id();
+
+ // get the chunk count
+ let stbl_box = &track.trak.mdia.minf.stbl;
+ let chunk_cnt = match &stbl_box.stco {
+ Some(stco) => stco.entries.len(),
+ None => match &stbl_box.co64 {
+ Some(co64) => co64.entries.len(),
+ None => 0,
+ },
+ };
+
+ // the Merkle count is the number of chunks for timed media
+ if mm.count != chunk_cnt as u32 {
+ return Err(Error::HashMismatch(
+ "Track count does not match Merkle map count".to_string(),
+ ));
+ }
+
+ // create sample to chunk mapping
+ // create the Merkle tree per samples in a chunk
+ let mut chunk_hash_map: HashMap<u32, Hasher> = HashMap::new();
+ let stsc = &track.trak.mdia.minf.stbl.stsc;
+ for sample_id in 1..=sample_cnt {
+ let stsc_idx = stsc_index(&track, sample_id)?;
+
+ let stsc_entry = &stsc.entries[stsc_idx];
+
+ let first_chunk = stsc_entry.first_chunk;
+ let first_sample = stsc_entry.first_sample;
+ let samples_per_chunk = stsc_entry.samples_per_chunk;
+
+ let chunk_id =
+ first_chunk + (sample_id - first_sample) / samples_per_chunk;
+
+ // add chunk Hasher if needed
+ if let Vacant(e) = chunk_hash_map.entry(chunk_id) {
+ // get hasher for algorithm
+ let hasher_enum = match alg.as_str() {
+ "sha256" => Hasher::SHA256(Sha256::new()),
+ "sha384" => Hasher::SHA384(Sha384::new()),
+ "sha512" => Hasher::SHA512(Sha512::new()),
+ _ => {
+ return Err(Error::HashMismatch(
+ "no algorithm found".to_string(),
+ ))
+ }
+ };
+
+ e.insert(hasher_enum);
+ }
+
+ if let Ok(Some(sample)) = &mp4.read_sample(track_id, sample_id) {
+ let h = chunk_hash_map.get_mut(&chunk_id).ok_or(
+ Error::HashMismatch(
+ "Bad Merkle tree sample mapping".to_string(),
+ ),
+ )?;
+ // add sample data to hash
+ h.update(&sample.bytes);
+ } else {
+ return Err(Error::HashMismatch(
+ "Merle location not found".to_owned(),
+ ));
+ }
+ }
+
+ if chunk_cnt != chunk_hash_map.len() {
+ return Err(Error::HashMismatch(
+ "Incorrect number of Merkle trees".to_string(),
+ ));
+ }
+
+ // finalize leaf hashes
+ let mut leaf_hashes = Vec::new();
+ for chunk_bmff_mm in &track_to_bmff_merkle_map[&track_id] {
+ match chunk_hash_map.remove(&(chunk_bmff_mm.location + 1)) {
+ Some(h) => {
+ let h = Hasher::finalize(h);
+ leaf_hashes.push(h.clone());
+ }
+ None => {
+ return Err(Error::HashMismatch(
+ "Could not generate hash".to_owned(),
+ ))
+ }
+ }
+ }
+
+ for chunk_bmff_mm in &track_to_bmff_merkle_map[&track_id] {
+ let hash = &leaf_hashes[chunk_bmff_mm.location as usize];
+
+ // check MerkleMap for the hash
+ if !mm.check_merkle_tree(
+ alg,
+ hash,
+ chunk_bmff_mm.location,
+ &chunk_bmff_mm.hashes,
+ ) {
+ return Err(Error::HashMismatch("Fragment not valid".to_string()));
+ }
+ }
+ }
+ }
+ } else {
+ // non-timed media so use iloc (awaiting use case/example since the iloc varies by format)
+ return Err(Error::HashMismatch(
+ "Merkle iloc not yet supported".to_owned(),
+ ));
+ }
}
- }
- pub fn verify_hash(&self, asset_path: &Path, alg: Option<&str>) -> Result<()> {
- let curr_alg = alg.unwrap_or("sha256");
+ Ok(())
+ }
- let bmff_exclusions = &self.exclusions;
+ // Used to verify fragmented BMFF assets spread across multiple file.
+ pub fn verify_stream_segment(
+ &self,
+ init_stream: &mut dyn CAIRead,
+ fragment_stream: &mut dyn CAIRead,
+ alg: Option<&str>,
+ ) -> crate::Result<()> {
+ let curr_alg = match &self.alg {
+ Some(a) => a.clone(),
+ None => match alg {
+ Some(a) => a.to_owned(),
+ None => "sha256".to_string(),
+ },
+ };
- // convert BMFF exclusion map to flat exclusion list
- let mut data = fs::File::open(asset_path)?;
- let exclusions =
- bmff_to_jumbf_exclusions(&mut data, bmff_exclusions, self.bmff_version > 1)?;
+ // handle file level hashing
+ if self.hash().is_some() {
+ return Err(Error::HashMismatch(
+ "Hash value should not be present for a fragmented BMFF asset".to_string(),
+ ));
+ }
- if verify_asset_by_alg(curr_alg, &self.hash, asset_path, Some(exclusions)) {
- Ok(())
+ // Merkle hashed BMFF
+ if let Some(mm_vec) = self.merkle() {
+ // get merkle boxes from segment
+ let c2pa_boxes = read_bmff_c2pa_boxes(fragment_stream)?;
+ let bmff_merkle = c2pa_boxes.bmff_merkle;
+
+ if bmff_merkle.is_empty() {
+ return Err(Error::HashMismatch("Fragment had no MerkleMap".to_string()));
+ }
+
+ for bmff_mm in bmff_merkle {
+ // find matching MerkleMap for this uniqueId & localId
+ if let Some(mm) = mm_vec
+ .iter()
+ .find(|mm| mm.unique_id == bmff_mm.unique_id && mm.local_id == bmff_mm.local_id)
+ {
+ let alg = match &mm.alg {
+ Some(a) => a,
+ None => &curr_alg,
+ };
+
+ // check the inithash (for fragmented MP4 with multiple files this is the hash of the init_segment minus any exclusions)
+ if let Some(init_hash) = &mm.init_hash {
+ let bmff_exclusions = &self.exclusions;
+
+ // convert BMFF exclusion map to flat exclusion list
+ init_stream.rewind()?;
+ let exclusions = bmff_to_jumbf_exclusions(
+ init_stream,
+ bmff_exclusions,
+ self.bmff_version > 1,
+ )?;
+
+ if !verify_stream_by_alg(
+ alg,
+ init_hash,
+ init_stream,
+ Some(exclusions),
+ true,
+ ) {
+ return Err(Error::HashMismatch("BMFF inithash mismatch".to_string()));
+ }
+
+ let fragment_exclusions = bmff_to_jumbf_exclusions(
+ fragment_stream,
+ bmff_exclusions,
+ self.bmff_version > 1,
+ )?;
+
+ // hash the entire fragment minus exclusions
+ let hash = hash_stream_by_alg(
+ alg,
+ fragment_stream,
+ Some(fragment_exclusions),
+ true,
+ )?;
+
+ // check MerkleMap for the hash
+ if !mm.check_merkle_tree(alg, &hash, bmff_mm.location, &bmff_mm.hashes) {
+ return Err(Error::HashMismatch("Fragment not valid".to_string()));
+ }
+ }
+ } else {
+ return Err(Error::HashMismatch("Fragment had no MerkleMap".to_string()));
+ }
+ }
} else {
- Err(Error::HashMismatch("Hashes do not match".to_owned()))
+ return Err(Error::HashMismatch(
+ "Merkle value must be present for a fragmented BMFF asset".to_string(),
+ ));
}
+
+ Ok(())
}
}
@@ -274,14 +841,104 @@ impl AssertionBase for BmffHash {
// todo: this mechanism needs to change since a struct could support different versions
- fn to_assertion(&self) -> Result<Assertion> {
+ fn to_assertion(&self) -> crate::error::Result<Assertion> {
Self::to_cbor_assertion(self)
}
- fn from_assertion(assertion: &Assertion) -> Result<Self> {
+ fn from_assertion(assertion: &Assertion) -> crate::error::Result<Self> {
let mut bmff_hash = Self::from_cbor_assertion(assertion)?;
bmff_hash.set_bmff_version(assertion.get_ver().unwrap_or(1));
Ok(bmff_hash)
}
}
+
+fn stsc_index(track: &Mp4Track, sample_id: u32) -> crate::Result<usize> {
+ if track.trak.mdia.minf.stbl.stsc.entries.is_empty() {
+ return Err(Error::InvalidAsset("BMFF has no stsc entries".to_string()));
+ }
+ for (i, entry) in track.trak.mdia.minf.stbl.stsc.entries.iter().enumerate() {
+ if sample_id < entry.first_sample {
+ return if i == 0 {
+ Err(Error::InvalidAsset("BMFF no sample not found".to_string()))
+ } else {
+ Ok(i - 1)
+ };
+ }
+ }
+ Ok(track.trak.mdia.minf.stbl.stsc.entries.len() - 1)
+}
+
+fn stream_len(reader: &mut dyn CAIRead) -> crate::Result<u64> {
+ let old_pos = reader.stream_position()?;
+ let len = reader.seek(SeekFrom::End(0))?;
+
+ if old_pos != len {
+ reader.seek(SeekFrom::Start(old_pos))?;
+ }
+
+ Ok(len)
+}
+
+/* we need shippable examples
+#[cfg(test)]
+pub mod tests {
+ #![allow(clippy::expect_used)]
+ #![allow(clippy::panic)]
+ #![allow(clippy::unwrap_used)]
+
+ //use tempfile::tempdir;
+
+ //use super::*;
+ use crate::utils::test::fixture_path;
+
+ #[cfg(not(target_arch = "wasm32"))]
+ #[test]
+ fn test_fragemented_mp4() {
+ use crate::{
+ assertions::BmffHash, asset_handlers::bmff_io::BmffIO, asset_io::AssetIO,
+ status_tracker::DetailedStatusTracker, store::Store, AssertionBase,
+ };
+
+ let init_stream_path = fixture_path("dashinit.mp4");
+ let segment_stream_path = fixture_path("dash1.m4s");
+ let segment_stream_path10 = fixture_path("dash10.m4s");
+ let segment_stream_path11 = fixture_path("dash11.m4s");
+
+
+ let mut init_stream = std::fs::File::open(init_stream_path).unwrap();
+ let mut segment_stream = std::fs::File::open(segment_stream_path).unwrap();
+ let mut segment_stream10 = std::fs::File::open(segment_stream_path10).unwrap();
+ let mut segment_stream11 = std::fs::File::open(segment_stream_path11).unwrap();
+
+
+ let mut log = DetailedStatusTracker::default();
+
+ let bmff_io = BmffIO::new("mp4");
+ let bmff_handler = bmff_io.get_reader();
+
+ let manifest_bytes = bmff_handler.read_cai(&mut init_stream).unwrap();
+ let store = Store::from_jumbf(&manifest_bytes, &mut log).unwrap();
+
+ // get the bmff hashes
+ let claim = store.provenance_claim().unwrap();
+ for dh_assertion in claim.data_hash_assertions() {
+ if dh_assertion.label_root() == BmffHash::LABEL {
+ let bmff_hash = BmffHash::from_assertion(dh_assertion).unwrap();
+
+ bmff_hash
+ .verify_stream_segment(&mut init_stream, &mut segment_stream, None)
+ .unwrap();
+
+ bmff_hash
+ .verify_stream_segment(&mut init_stream, &mut segment_stream10, None)
+ .unwrap();
+
+ bmff_hash
+ .verify_stream_segment(&mut init_stream, &mut segment_stream11, None)
+ .unwrap();
+ }
+ }
+ }
+}
+*/
diff --git a/sdk/src/assertions/data_hash.rs b/sdk/src/assertions/data_hash.rs
@@ -22,9 +22,12 @@ use serde_bytes::ByteBuf;
use crate::{
assertion::{Assertion, AssertionBase, AssertionCbor},
assertions::labels,
+ asset_io::CAIRead,
cbor_types::UriT,
error::{Error, Result},
- utils::hash_utils::{hash_stream_by_alg, verify_asset_by_alg, verify_by_alg, Exclusion},
+ utils::hash_utils::{
+ hash_stream_by_alg, verify_asset_by_alg, verify_by_alg, verify_stream_by_alg, HashRange,
+ },
};
const ASSERTION_CREATION_VERSION: usize = 1;
@@ -33,7 +36,7 @@ const ASSERTION_CREATION_VERSION: usize = 1;
#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)]
pub struct DataHash {
#[serde(skip_serializing_if = "Option::is_none")]
- pub exclusions: Option<Vec<Exclusion>>,
+ pub exclusions: Option<Vec<HashRange>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
@@ -77,7 +80,7 @@ impl DataHash {
}
}
- pub fn add_exclusion(&mut self, exclusion: Exclusion) {
+ pub fn add_exclusion(&mut self, exclusion: HashRange) {
if self.exclusions.is_none() {
self.exclusions = Some(Vec::new());
}
@@ -178,8 +181,8 @@ impl DataHash {
// sort the exclusions
let hash = match self.exclusions {
- Some(ref e) => hash_stream_by_alg(&alg, stream, Some(e.clone()))?,
- None => hash_stream_by_alg(&alg, stream, None)?,
+ Some(ref e) => hash_stream_by_alg(&alg, stream, Some(e.clone()), true)?,
+ None => hash_stream_by_alg(&alg, stream, None, true)?,
};
if hash.is_empty() {
@@ -190,7 +193,7 @@ impl DataHash {
}
// verify data using currently set algorithm or default alg is none currently set
- pub fn verify_in_memory_hash(&self, data: &[u8], alg: Option<String>) -> Result<()> {
+ pub fn verify_in_memory_hash(&self, data: &[u8], alg: Option<&str>) -> Result<()> {
if self.is_remote_hash() {
return Err(Error::BadParam("asset hash is remote".to_owned()));
}
@@ -198,7 +201,7 @@ impl DataHash {
let curr_alg = match &self.alg {
Some(a) => a.clone(),
None => match alg {
- Some(a) => a,
+ Some(a) => a.to_owned(),
None => "sha256".to_string(),
},
};
@@ -230,6 +233,29 @@ impl DataHash {
}
}
+ // verify data using currently set algorithm or default alg is none currently set
+ pub fn verify_stream_hash(&self, reader: &mut dyn CAIRead, alg: Option<&str>) -> Result<()> {
+ if self.is_remote_hash() {
+ return Err(Error::BadParam("asset hash is remote".to_owned()));
+ }
+
+ let curr_alg = match &self.alg {
+ Some(a) => a.clone(),
+ None => match alg {
+ Some(a) => a.to_owned(),
+ None => "sha256".to_string(),
+ },
+ };
+
+ let exclusions = self.exclusions.as_ref().cloned();
+
+ if verify_stream_by_alg(&curr_alg, &self.hash, reader, exclusions, true) {
+ Ok(())
+ } else {
+ Err(Error::HashMismatch("Hashes do not match".to_owned()))
+ }
+ }
+
/// Create a new instance from Assertion
pub fn from_assertion(assertion: &Assertion) -> Result<Self> {
assertion.check_version_from_label(ASSERTION_CREATION_VERSION)?;
@@ -272,7 +298,7 @@ pub mod tests {
fn test_build_assertion() {
// try json based assertion
let mut data_hash = DataHash::new("Some data", "sha256", None);
- data_hash.add_exclusion(Exclusion::new(0, 1234));
+ data_hash.add_exclusion(HashRange::new(0, 1234));
data_hash.hash = vec![1, 2, 3];
let assertion = data_hash.to_assertion().unwrap();
@@ -313,8 +339,8 @@ pub mod tests {
#[test]
fn test_binary_round_trip() {
let mut data_hash = DataHash::new("Some data", "sha256", None);
- data_hash.add_exclusion(Exclusion::new(0x2000, 0x1000));
- data_hash.add_exclusion(Exclusion::new(0x4000, 0x1000));
+ data_hash.add_exclusion(HashRange::new(0x2000, 0x1000));
+ data_hash.add_exclusion(HashRange::new(0x4000, 0x1000));
// add some data to hash
let ap = fixture_path("earth_apollo17.jpg");
diff --git a/sdk/src/assertions/mod.rs b/sdk/src/assertions/mod.rs
@@ -17,7 +17,7 @@ mod actions;
pub use actions::{c2pa_action, Action, Actions};
mod bmff_hash;
-pub use bmff_hash::{BmffHash, DataMap, ExclusionsMap, SubsetMap};
+pub use bmff_hash::{BmffHash, BmffMerkleMap, DataMap, ExclusionsMap, SubsetMap};
#[allow(dead_code)] // will become public later
mod data_hash;
diff --git a/sdk/src/asset_handlers/bmff_io.rs b/sdk/src/asset_handlers/bmff_io.rs
@@ -12,6 +12,7 @@
// each license.
use std::{
+ cmp::min,
collections::HashMap,
convert::{From, TryFrom},
fs::{File, OpenOptions},
@@ -22,18 +23,16 @@ use std::{
use atree::{Arena, Token};
use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
use conv::ValueFrom;
-use serde::{Deserialize, Serialize};
-use serde_bytes::ByteBuf;
use tempfile::Builder;
use crate::{
- assertions::ExclusionsMap,
+ assertions::{BmffMerkleMap, ExclusionsMap},
asset_io::{
AssetIO, AssetPatch, CAIRead, CAIReadWrite, CAIReader, HashObjectPositions, RemoteRefEmbed,
RemoteRefEmbedType,
},
error::{Error, Result},
- utils::hash_utils::{vec_compare, Exclusion},
+ utils::hash_utils::{vec_compare, HashRange},
};
pub struct BmffIO {
@@ -161,16 +160,6 @@ boxtype! {
IlocBox => 0x696C6F63
}
-#[derive(Serialize, Deserialize, Debug, PartialEq)]
-struct BmffMerkleMap {
- #[serde(rename = "uniqueId")]
- unique_id: u32,
- #[serde(rename = "localId")]
- local_id: u32,
- location: u32,
- hashes: Option<Vec<ByteBuf>>,
-}
-
struct BoxHeaderLite {
pub name: BoxType,
pub size: u64,
@@ -244,18 +233,25 @@ fn write_box_uuid_extension<W: Write>(w: &mut W, uuid: &[u8; 16]) -> Result<u64>
Ok(16)
}
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
pub(crate) struct BoxInfo {
path: String,
parent: Option<Token>,
- offset: u64,
- size: u64,
+ pub offset: u64,
+ pub size: u64,
box_type: BoxType,
user_type: Option<Vec<u8>>,
version: Option<u8>,
flags: Option<u32>,
}
+#[derive(Clone, Debug, PartialEq)]
+pub(crate) struct BoxInfoLite {
+ pub path: String,
+ pub offset: u64,
+ pub size: u64,
+}
+
fn read_box_header_ext(reader: &mut dyn CAIRead) -> Result<(u8, u32)> {
let version = reader.read_u8()?;
let flags = reader.read_u24::<BigEndian>()?;
@@ -393,14 +389,37 @@ fn get_top_level_box_offsets(
tl_offsets
}
+fn get_top_level_boxes(
+ bmff_tree: &Arena<BoxInfo>,
+ bmff_path_map: &HashMap<String, Vec<Token>>,
+) -> Vec<BoxInfoLite> {
+ let mut tl_boxes = Vec::new();
+
+ for (p, t) in bmff_path_map {
+ // look for top level offsets
+ if p.matches('/').count() == 1 {
+ for token in t {
+ if let Some(box_info) = bmff_tree.get(*token) {
+ tl_boxes.push(BoxInfoLite {
+ path: box_info.data.path.clone(),
+ offset: box_info.data.offset,
+ size: box_info.data.size,
+ });
+ }
+ }
+ }
+ }
+
+ tl_boxes
+}
+
pub fn bmff_to_jumbf_exclusions(
reader: &mut dyn CAIRead,
bmff_exclusions: &[ExclusionsMap],
bmff_v2: bool,
-) -> Result<Vec<Exclusion>> {
- let start = reader.stream_position()?;
+) -> Result<Vec<HashRange>> {
let size = reader.seek(SeekFrom::End(0))?;
- reader.seek(SeekFrom::Start(start))?;
+ reader.rewind()?;
// create root node
let root_box = BoxInfo {
@@ -511,12 +530,12 @@ pub fn bmff_to_jumbf_exclusions(
// reduce range if desired
if let Some(subset_vec) = &bmff_exclusion.subset {
for subset in subset_vec {
- let exclusion = Exclusion::new(
+ let exclusion = HashRange::new(
(exclusion_start + subset.offset as u64) as usize,
(if subset.length == 0 {
exclusion_length - subset.offset as u64
} else {
- subset.length as u64
+ min(subset.length as u64, exclusion_length)
}) as usize,
);
@@ -525,7 +544,7 @@ pub fn bmff_to_jumbf_exclusions(
} else {
// exclude box in its entirty
let exclusion =
- Exclusion::new(exclusion_start as usize, exclusion_length as usize);
+ HashRange::new(exclusion_start as usize, exclusion_length as usize);
exclusions.push(exclusion);
@@ -543,7 +562,7 @@ pub fn bmff_to_jumbf_exclusions(
// note: this is technically not an exclusion but a replacement with a new range of bytes to be hashed
if bmff_v2 {
for tl_start in tl_offsets {
- let mut exclusion = Exclusion::new(tl_start as usize, 1);
+ let mut exclusion = HashRange::new(tl_start as usize, 1);
exclusion.set_bmff_offset(tl_start);
exclusions.push(exclusion);
@@ -1018,105 +1037,138 @@ fn get_manifest_token(
None
}
-impl CAIReader for BmffIO {
- fn read_cai(&self, reader: &mut dyn CAIRead) -> Result<Vec<u8>> {
- let start = reader.stream_position()?;
- let size = reader.seek(SeekFrom::End(0))?;
- reader.seek(SeekFrom::Start(start))?;
+pub(crate) struct C2PABmffBoxes {
+ pub manifest_bytes: Option<Vec<u8>>,
+ pub bmff_merkle: Vec<BmffMerkleMap>,
+ pub box_infos: Vec<BoxInfoLite>,
+}
- // create root node
- let root_box = BoxInfo {
- path: "".to_string(),
- offset: 0,
- size,
- box_type: BoxType::Empty,
- parent: None,
- user_type: None,
- version: None,
- flags: None,
- };
+pub(crate) fn read_bmff_c2pa_boxes(reader: &mut dyn CAIRead) -> Result<C2PABmffBoxes> {
+ let size = reader.seek(SeekFrom::End(0))?;
+ reader.rewind()?;
- let (mut bmff_tree, root_token) = Arena::with_data(root_box);
- let mut bmff_map: HashMap<String, Vec<Token>> = HashMap::new();
+ // create root node
+ let root_box = BoxInfo {
+ path: "".to_string(),
+ offset: 0,
+ size,
+ box_type: BoxType::Empty,
+ parent: None,
+ user_type: None,
+ version: None,
+ flags: None,
+ };
- // build layout of the BMFF structure
- build_bmff_tree(reader, size, &mut bmff_tree, &root_token, &mut bmff_map)?;
+ let (mut bmff_tree, root_token) = Arena::with_data(root_box);
+ let mut bmff_map: HashMap<String, Vec<Token>> = HashMap::new();
- let mut output: Option<Vec<u8>> = None;
+ // build layout of the BMFF structure
+ build_bmff_tree(reader, size, &mut bmff_tree, &root_token, &mut bmff_map)?;
+
+ let mut output: Option<Vec<u8>> = None;
+ let mut _first_aux_uuid = 0;
+ let mut merkle_boxes: Vec<BmffMerkleMap> = Vec::new();
+
+ // grab top level (for now) C2PA box
+ if let Some(uuid_list) = bmff_map.get("/uuid") {
+ let mut manifest_store_cnt = 0;
+
+ for uuid_token in uuid_list {
+ let box_info = &bmff_tree[*uuid_token];
+
+ // make sure it is UUID box
+ if box_info.data.box_type == BoxType::UuidBox {
+ if let Some(uuid) = &box_info.data.user_type {
+ // make sure it is a C2PA ContentProvenanceBox box
+ if vec_compare(&C2PA_UUID, uuid) {
+ let mut data_len = box_info.data.size - HEADER_SIZE - 16 /*UUID*/;
+
+ // set reader to start of box contents
+ skip_bytes_to(reader, box_info.data.offset + HEADER_SIZE + 16)?;
+
+ // Fullbox => 8 bits for version 24 bits for flags
+ let (_version, _flags) = read_box_header_ext(reader)?;
+ data_len -= 4;
+
+ // get the purpose
+ let mut purpose = Vec::with_capacity(64);
+ loop {
+ let mut buf = [0; 1];
+ reader.read_exact(&mut buf)?;
+ data_len -= 1;
+ if buf[0] == 0x00 {
+ break;
+ } else {
+ purpose.push(buf[0]);
+ }
+ }
- // grab top level (for now) C2PA box
- if let Some(uuid_list) = bmff_map.get("/uuid") {
- let mut manifest_store_cnt = 0;
+ // is the purpose manifest?
+ if vec_compare(&purpose, MANIFEST.as_bytes()) {
+ // offset to first aux uuid with purpose merkle
+ let mut buf = [0u8; 8];
+ reader.read_exact(&mut buf)?;
+ data_len -= 8;
- for uuid_token in uuid_list {
- let box_info = &bmff_tree[*uuid_token];
+ // offset to first aux uuid
+ let offset = u64::from_be_bytes(buf);
- // make sure it is UUID box
- if box_info.data.box_type == BoxType::UuidBox {
- if let Some(uuid) = &box_info.data.user_type {
- // make sure it is a C2PA ContentProvenanceBox box
- if vec_compare(&C2PA_UUID, uuid) {
- let mut data_len = box_info.data.size - HEADER_SIZE - 16 /*UUID*/;
+ // read the manifest
+ if manifest_store_cnt == 0 {
+ let mut manifest = vec![0u8; data_len as usize];
+ reader.read_exact(&mut manifest)?;
+ output = Some(manifest);
- // set reader to start of box contents
- skip_bytes_to(reader, box_info.data.offset + HEADER_SIZE + 16)?;
+ manifest_store_cnt += 1;
+ } else {
+ return Err(Error::TooManyManifestStores);
+ }
- // Fullbox => 8 bits for version 24 bits for flags
- let (_version, _flags) = read_box_header_ext(reader)?;
- data_len -= 4;
+ // if contains offset this asset contains additional UUID boxes
+ if offset != 0 {
+ _first_aux_uuid = offset;
+ }
+ } else if vec_compare(&purpose, MERKLE.as_bytes()) {
+ let mut merkle = vec![0u8; data_len as usize];
+ reader.read_exact(&mut merkle)?;
- // get the purpose
- let mut purpose = Vec::with_capacity(64);
+ // strip trailing zeros
loop {
- let mut buf = [0; 1];
- reader.read_exact(&mut buf)?;
- data_len -= 1;
- if buf[0] == 0x00 {
- break;
- } else {
- purpose.push(buf[0]);
+ if !merkle.is_empty() && merkle[merkle.len() - 1] == 0 {
+ merkle.pop();
}
- }
- // is the purpose manifest?
- if vec_compare(&purpose, MANIFEST.as_bytes()) {
- // offset to first aux uuid with purpose merkle
- let mut buf = [0u8; 8];
- reader.read_exact(&mut buf)?;
- data_len -= 8;
-
- // offset to first aux uuid
- let offset = u64::from_be_bytes(buf);
-
- // if no offset this contains the manifest
- if offset == 0 {
- if manifest_store_cnt == 0 {
- let mut manifest = vec![0u8; data_len as usize];
- reader.read_exact(&mut manifest)?;
- output = Some(manifest);
-
- manifest_store_cnt += 1;
- } else {
- return Err(Error::TooManyManifestStores);
- }
- } else {
- // handle aux uuids
- let mut buf = vec![0u8; data_len as usize];
- reader.read_exact(&mut buf)?;
-
- let _mm: BmffMerkleMap = serde_cbor::from_slice(&buf)?;
+ if merkle.is_empty() || merkle[merkle.len() - 1] != 0 {
+ break;
}
- } else if vec_compare(&purpose, MERKLE.as_bytes()) {
- // handle merkle boxes not yet handled
- return Err(Error::UnsupportedType);
}
+
+ // find uuid from uuid list
+ let mm: BmffMerkleMap = serde_cbor::from_slice(&merkle)?;
+ merkle_boxes.push(mm);
}
}
}
}
}
+ }
+
+ // get position ordered list of boxes
+ let mut box_infos: Vec<BoxInfoLite> = get_top_level_boxes(&bmff_tree, &bmff_map);
+ box_infos.sort_by(|a, b| a.offset.cmp(&b.offset));
+
+ Ok(C2PABmffBoxes {
+ manifest_bytes: output,
+ bmff_merkle: merkle_boxes,
+ box_infos,
+ })
+}
+
+impl CAIReader for BmffIO {
+ fn read_cai(&self, reader: &mut dyn CAIRead) -> Result<Vec<u8>> {
+ let c2pa_boxes = read_bmff_c2pa_boxes(reader)?;
- output.ok_or(Error::JumbfNotFound)
+ c2pa_boxes.manifest_bytes.ok_or(Error::JumbfNotFound)
}
// Get XMP block
diff --git a/sdk/src/claim.rs b/sdk/src/claim.rs
@@ -28,6 +28,7 @@ use crate::{
labels::{self, CLAIM},
BmffHash, DataHash,
},
+ asset_io::CAIRead,
cose_validator::{get_signing_info, verify_cose, verify_cose_async},
error::{Error, Result},
hashed_uri::HashedUri,
@@ -53,9 +54,14 @@ use HashedUri as C2PAAssertion;
const GH_FULL_VERSION_LIST: &str = "Sec-CH-UA-Full-Version-List";
const GH_UA: &str = "Sec-CH-UA";
+// Enum to encapsulate the data type of the source asset. This simplifies
+// having different implementations for functions as a single entry point can be
+// used to handle different data types.
pub enum ClaimAssetData<'a> {
- PathData(&'a Path),
- ByteData(&'a [u8]),
+ Path(&'a Path),
+ Bytes(&'a [u8]),
+ Stream(&'a mut dyn CAIRead),
+ StreamFragment(&'a mut dyn CAIRead, &'a mut dyn CAIRead),
}
#[derive(PartialEq, Eq, Clone)]
@@ -862,7 +868,7 @@ impl Claim {
/// asset_bytes - reference to bytes of the asset
pub async fn verify_claim_async<'a>(
claim: &Claim,
- asset_bytes: &'a [u8],
+ asset_data: &mut ClaimAssetData<'_>,
is_provenance: bool,
validation_log: &mut impl StatusTracker,
) -> Result<()> {
@@ -900,13 +906,7 @@ impl Claim {
validation_log,
)
.await;
- Claim::verify_internal(
- claim,
- &ClaimAssetData::ByteData(asset_bytes),
- is_provenance,
- verified,
- validation_log,
- )
+ Claim::verify_internal(claim, asset_data, is_provenance, verified, validation_log)
}
/// Verify claim signature, assertion store and asset hashes
@@ -914,7 +914,7 @@ impl Claim {
/// asset_bytes - reference to bytes of the asset
pub fn verify_claim(
claim: &Claim,
- asset_data: &ClaimAssetData<'_>,
+ asset_data: &mut ClaimAssetData<'_>,
is_provenance: bool,
validation_log: &mut impl StatusTracker,
) -> Result<()> {
@@ -962,7 +962,7 @@ impl Claim {
fn verify_internal(
claim: &Claim,
- asset_data: &ClaimAssetData<'_>,
+ asset_data: &mut ClaimAssetData<'_>,
is_provenance: bool,
verified: Result<ValidationInfo>,
validation_log: &mut impl StatusTracker,
@@ -1043,6 +1043,7 @@ impl Claim {
.validation_status(validation_status::MANIFEST_UPDATE_INVALID);
validation_log.log(log_item, Some(Error::UpdateManifestInvalid))?;
}
+
// verify assertion structure comparing hashes from assertion list to contents of assertion store
for assertion in claim.assertions() {
let (label, instance) = Claim::assertion_label_from_link(&assertion.url());
@@ -1130,12 +1131,16 @@ impl Claim {
if !dh.is_remote_hash() {
// only verify local hashes here
let hash_result = match asset_data {
- ClaimAssetData::PathData(asset_path) => {
+ ClaimAssetData::Path(asset_path) => {
dh.verify_hash(asset_path, Some(claim.alg()))
}
- ClaimAssetData::ByteData(asset_bytes) => {
- dh.verify_in_memory_hash(asset_bytes, Some(claim.alg().to_string()))
+ ClaimAssetData::Bytes(asset_bytes) => {
+ dh.verify_in_memory_hash(asset_bytes, Some(claim.alg()))
}
+ ClaimAssetData::Stream(stream_data) => {
+ dh.verify_stream_hash(*stream_data, Some(claim.alg()))
+ }
+ _ => return Err(Error::UnsupportedType), /* this should never happen (coding error) */
};
match hash_result {
@@ -1166,19 +1171,28 @@ impl Claim {
}
}
}
- } else {
+ } else if dh_assertion.label_root() == BmffHash::LABEL {
// handle BMFF data hashes
let dh = BmffHash::from_assertion(dh_assertion)?;
let name = dh.name().map_or("unnamed".to_string(), default_str);
let hash_result = match asset_data {
- ClaimAssetData::PathData(asset_path) => {
+ ClaimAssetData::Path(asset_path) => {
dh.verify_hash(asset_path, Some(claim.alg()))
}
- ClaimAssetData::ByteData(asset_bytes) => {
- dh.verify_in_memory_hash(asset_bytes, Some(claim.alg().to_string()))
+ ClaimAssetData::Bytes(asset_bytes) => {
+ dh.verify_in_memory_hash(asset_bytes, Some(claim.alg()))
}
+ ClaimAssetData::Stream(stream_data) => {
+ dh.verify_stream(*stream_data, Some(claim.alg()))
+ }
+ ClaimAssetData::StreamFragment(initseg_data, fragment_data) => dh
+ .verify_stream_segment(
+ *initseg_data,
+ *fragment_data,
+ Some(claim.alg()),
+ ),
};
match hash_result {
@@ -1208,6 +1222,9 @@ impl Claim {
)?;
}
}
+ } else {
+ // box hash case
+ return Err(Error::UnsupportedType); // implementation to come
}
}
}
diff --git a/sdk/src/ingredient.rs b/sdk/src/ingredient.rs
@@ -24,7 +24,7 @@ use crate::{
assertion::{get_thumbnail_image_type, Assertion, AssertionBase},
assertions::{self, labels, Metadata, Relationship, Thumbnail},
asset_io::CAIRead,
- claim::Claim,
+ claim::{Claim, ClaimAssetData},
error::{Error, Result},
hashed_uri::HashedUri,
jumbf,
@@ -740,13 +740,13 @@ impl Ingredient {
match Store::from_jumbf(&manifest_bytes, &mut validation_log) {
Ok(store) => {
// verify the store
- //todo, change this when we have a stream version of verify
- let mut buf: Vec<u8> = Vec::new();
- stream.rewind()?;
- stream.read_to_end(&mut buf).map_err(Error::IoError)?;
- Store::verify_store_async(&store, &buf, &mut validation_log)
- .await
- .map(|_| store)
+ Store::verify_store_async(
+ &store,
+ &mut ClaimAssetData::Stream(stream),
+ &mut validation_log,
+ )
+ .await
+ .map(|_| store)
}
Err(e) => {
validation_log.log_silent(
@@ -1019,7 +1019,6 @@ impl Ingredient {
stream: &mut dyn CAIRead,
) -> Result<Self> {
let mut ingredient = Self::from_stream_info(stream, format, "untitled");
- stream.rewind()?;
let mut validation_log = DetailedStatusTracker::new();
@@ -1028,13 +1027,14 @@ impl Ingredient {
let result = match Store::from_jumbf(&manifest_bytes, &mut validation_log) {
Ok(store) => {
// verify the store
- //todo, change this when we have a stream version of verify
- let mut buf: Vec<u8> = Vec::new();
stream.rewind()?;
- stream.read_to_end(&mut buf).map_err(Error::IoError)?;
- Store::verify_store_async(&store, &buf, &mut validation_log)
- .await
- .map(|_| store)
+ Store::verify_store_async(
+ &store,
+ &mut ClaimAssetData::Stream(stream),
+ &mut validation_log,
+ )
+ .await
+ .map(|_| store)
}
Err(e) => {
// add a log entry for the error so we act like verify
diff --git a/sdk/src/manifest_store.rs b/sdk/src/manifest_store.rs
@@ -18,6 +18,7 @@ use std::path::Path;
use serde::Serialize;
use crate::{
+ claim::ClaimAssetData,
status_tracker::{DetailedStatusTracker, StatusTracker},
store::Store,
validation_status::{status_for_store, ValidationStatus},
@@ -218,6 +219,28 @@ impl ManifestStore {
.map(|store| Self::from_store(&store, &mut validation_log))
}
+ /// Loads a ManifestStore from an init segment and fragment. This
+ /// would be used to load and validate fragmented MP4 files that span
+ /// multiple separate assets.
+ pub async fn from_fragment_bytes_async(
+ format: &str,
+ init_bytes: &[u8],
+ fragment_bytes: &[u8],
+ verify: bool,
+ ) -> Result<ManifestStore> {
+ let mut validation_log = DetailedStatusTracker::new();
+
+ Store::load_fragment_from_memory_async(
+ format,
+ init_bytes,
+ fragment_bytes,
+ verify,
+ &mut validation_log,
+ )
+ .await
+ .map(|store| Self::from_store(&store, &mut validation_log))
+ }
+
/// Asynchronously loads a manifest from a buffer holding a binary manifest (.c2pa) and validates against an asset buffer
///
/// # Example: Creating a manifest store from a .c2pa manifest and validating it against an asset
@@ -246,7 +269,12 @@ impl ManifestStore {
let mut validation_log = DetailedStatusTracker::new();
let store = Store::from_jumbf(manifest_bytes, &mut validation_log)?;
- Store::verify_store_async(&store, asset_bytes, &mut validation_log).await?;
+ Store::verify_store_async(
+ &store,
+ &mut ClaimAssetData::Bytes(asset_bytes),
+ &mut validation_log,
+ )
+ .await?;
Ok(Self::from_store(&store, &mut validation_log))
}
diff --git a/sdk/src/manifest_store_report.rs b/sdk/src/manifest_store_report.rs
@@ -188,10 +188,8 @@ impl ManifestStoreReport {
} else {
format!("Asset:{}, Manifest:{}", ingredient_assertion.title, label)
};
- let new_token = tree.new_node(data);
- current_token.append_node(tree, new_token).map_err(|_err| {
- crate::Error::InvalidAsset("Bad Manifest graph".to_string())
- })?;
+
+ let new_token = current_token.append(tree, data);
ManifestStoreReport::populate_node(
tree,
diff --git a/sdk/src/store.rs b/sdk/src/store.rs
@@ -47,7 +47,7 @@ use crate::{
},
status_tracker::{log_item, OneShotStatusTracker, StatusTracker},
utils::{
- hash_utils::{hash256, Exclusion},
+ hash_utils::{hash256, HashRange},
patch::patch_bytes,
},
validation_status, AsyncSigner, ManifestStoreReport, Signer,
@@ -1076,7 +1076,7 @@ impl Store {
fn ingredient_checks(
store: &Store,
claim: &Claim,
- asset_data: &ClaimAssetData<'_>,
+ asset_data: &mut ClaimAssetData<'_>,
validation_log: &mut impl StatusTracker,
) -> Result<()> {
let mut num_parent_ofs = 0;
@@ -1195,7 +1195,7 @@ impl Store {
async fn ingredient_checks_async(
store: &Store,
claim: &Claim,
- asset_bytes: &[u8],
+ asset_data: &mut ClaimAssetData<'_>,
validation_log: &mut impl StatusTracker,
) -> Result<()> {
// walk the ingredients
@@ -1240,7 +1240,7 @@ impl Store {
)?;
}
// verify the ingredient claim
- Claim::verify_claim_async(ingredient, asset_bytes, false, validation_log)
+ Claim::verify_claim_async(ingredient, asset_data, false, validation_log)
.await?;
} else {
let log_item = log_item!(
@@ -1272,7 +1272,7 @@ impl Store {
/// validation_log: If present all found errors are logged and returned, other wise first error causes exit and is returned
pub async fn verify_store_async(
store: &Store,
- asset_bytes: &[u8],
+ asset_data: &mut ClaimAssetData<'_>,
validation_log: &mut impl StatusTracker,
) -> Result<()> {
let claim = match store.provenance_claim() {
@@ -1289,9 +1289,9 @@ impl Store {
};
// verify the provenance claim
- Claim::verify_claim_async(claim, asset_bytes, true, validation_log).await?;
+ Claim::verify_claim_async(claim, asset_data, true, validation_log).await?;
- Store::ingredient_checks_async(store, claim, asset_bytes, validation_log).await?;
+ Store::ingredient_checks_async(store, claim, asset_data, validation_log).await?;
Ok(())
}
@@ -1303,7 +1303,7 @@ impl Store {
/// validation_log: If present all found errors are logged and returned, other wise first error causes exit and is returned
pub fn verify_store(
store: &Store,
- asset_data: &ClaimAssetData<'_>,
+ asset_data: &mut ClaimAssetData<'_>,
validation_log: &mut impl StatusTracker,
) -> Result<()> {
let claim = match store.provenance_claim() {
@@ -1385,7 +1385,7 @@ impl Store {
// add exclusion hash for bytes before and after jumbf
let mut dh = DataHash::new("jumbf manifest", alg, None);
if block_end > block_start {
- dh.add_exclusion(Exclusion::new(block_start, block_end - block_start));
+ dh.add_exclusion(HashRange::new(block_start, block_end - block_start));
}
if calc_hashes {
@@ -2153,7 +2153,7 @@ impl Store {
asset_path: &'_ Path,
validation_log: &mut impl StatusTracker,
) -> Result<()> {
- Store::verify_store(self, &ClaimAssetData::PathData(asset_path), validation_log)
+ Store::verify_store(self, &mut ClaimAssetData::Path(asset_path), validation_log)
}
// verify from a buffer without file i/o
@@ -2163,7 +2163,17 @@ impl Store {
_asset_type: &str,
validation_log: &mut impl StatusTracker,
) -> Result<()> {
- Store::verify_store(self, &ClaimAssetData::ByteData(buf), validation_log)
+ Store::verify_store(self, &mut ClaimAssetData::Bytes(buf), validation_log)
+ }
+
+ // verify from a buffer without file i/o
+ pub fn verify_from_stream(
+ &mut self,
+ reader: &mut dyn CAIRead,
+ _asset_type: &str,
+ validation_log: &mut impl StatusTracker,
+ ) -> Result<()> {
+ Store::verify_store(self, &mut ClaimAssetData::Stream(reader), validation_log)
}
// fetch remote manifest if possible
@@ -2421,7 +2431,7 @@ impl Store {
/// validation_log: If present all found errors are logged and returned, otherwise first error causes exit and is returned
pub fn load_from_memory(
asset_type: &str,
- data: &'_ [u8],
+ data: &[u8],
verify: bool,
validation_log: &mut impl StatusTracker,
) -> Result<Store> {
@@ -2429,7 +2439,7 @@ impl Store {
// verify the store
if verify {
// verify store and claims
- Store::verify_store(&store, &ClaimAssetData::ByteData(data), validation_log)?;
+ Store::verify_store(&store, &mut ClaimAssetData::Bytes(data), validation_log)?;
}
Ok(store)
@@ -2438,7 +2448,7 @@ impl Store {
/// Load Store from a in-memory asset asychronously validating
/// asset_type: asset extension or mime type
- /// data: reference to bytes of the the file
+ /// data: reference to bytes of the file
/// verify: if true will run verification checks when loading
/// validation_log: If present all found errors are logged and returned, otherwise first error causes exit and is returned
pub async fn load_from_memory_async(
@@ -2452,7 +2462,73 @@ impl Store {
// verify the store
if verify {
// verify store and claims
- Store::verify_store_async(&store, data, validation_log).await?;
+ Store::verify_store_async(&store, &mut ClaimAssetData::Bytes(data), validation_log)
+ .await?;
+ }
+
+ Ok(store)
+ }
+
+ /// Load Store from a in-memory asset
+ /// asset_type: asset extension or mime type
+ /// data: reference to bytes of the the file
+ /// verify: if true will run verification checks when loading
+ /// validation_log: If present all found errors are logged and returned, otherwise first error causes exit and is returned
+ pub fn load_fragment_from_memory(
+ asset_type: &str,
+ init_segment: &[u8],
+ fragment: &[u8],
+ verify: bool,
+ validation_log: &mut impl StatusTracker,
+ ) -> Result<Store> {
+ Store::get_store_from_memory(asset_type, init_segment, validation_log).and_then(|store| {
+ // verify the store
+ if verify {
+ let mut init_segment_stream = Cursor::new(init_segment);
+ let mut fragment_stream = Cursor::new(fragment);
+
+ // verify store and claims
+ Store::verify_store(
+ &store,
+ &mut ClaimAssetData::StreamFragment(
+ &mut init_segment_stream,
+ &mut fragment_stream,
+ ),
+ validation_log,
+ )?;
+ }
+
+ Ok(store)
+ })
+ }
+
+ /// Load Store from a in-memory asset asychronously validating
+ /// asset_type: asset extension or mime type
+ /// init_segment: reference to bytes of the init segment
+ /// fragment: reference to bytes of the fragment to validate
+ /// verify: if true will run verification checks when loading
+ /// validation_log: If present all found errors are logged and returned, otherwise first error causes exit and is returned
+ pub async fn load_fragment_from_memory_async(
+ asset_type: &str,
+ init_segment: &[u8],
+ fragment: &[u8],
+ verify: bool,
+ validation_log: &mut impl StatusTracker,
+ ) -> Result<Store> {
+ let store = Store::get_store_from_memory(asset_type, init_segment, validation_log)?;
+
+ // verify the store
+ if verify {
+ let mut init_segment_stream = Cursor::new(init_segment);
+ let mut fragment_stream = Cursor::new(fragment);
+
+ // verify store and claims
+ Store::verify_store_async(
+ &store,
+ &mut ClaimAssetData::StreamFragment(&mut init_segment_stream, &mut fragment_stream),
+ validation_log,
+ )
+ .await?;
}
Ok(store)
@@ -3725,6 +3801,29 @@ pub mod tests {
let store = Store::load_from_asset(&ap, true, &mut report).expect("load_from_asset");
println!("store = {store}");
}
+
+ /*
+ #[test]
+ fn test_bmff_fragments() {
+ let init_stream_path = fixture_path("dashinit.mp4");
+ let segment_stream_path = fixture_path("dash1.m4s");
+
+ let init_stream = std::fs::read(init_stream_path).unwrap();
+ let segment_stream = std::fs::read(segment_stream_path).unwrap();
+
+ let mut report = DetailedStatusTracker::new();
+ let store = Store::load_fragment_from_memory(
+ "mp4",
+ &init_stream,
+ &segment_stream,
+ true,
+ &mut report,
+ )
+ .expect("load_from_asset");
+ println!("store = {store}");
+ }
+ */
+
#[test]
fn test_bmff_jumbf_generation() {
// test adding to actual image
diff --git a/sdk/src/utils/hash_utils.rs b/sdk/src/utils/hash_utils.rs
@@ -33,7 +33,7 @@ use crate::{Error, Result};
const MAX_HASH_BUF: usize = 256 * 1024 * 1024; // cap memory usage to 256MB
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)]
-pub struct Exclusion {
+pub struct HashRange {
start: usize,
length: usize,
@@ -41,9 +41,9 @@ pub struct Exclusion {
bmff_offset: Option<u64>, /* optional tracking of offset positions to include in BMFF_V2 hashes in BE format */
}
-impl Exclusion {
+impl HashRange {
pub fn new(start: usize, length: usize) -> Self {
- Exclusion {
+ HashRange {
start,
length,
bmff_offset: None,
@@ -101,7 +101,7 @@ pub fn hash_by_type(hash_type: u8, data: &[u8]) -> Option<Multihash> {
}
#[derive(Clone)]
-enum Hasher {
+pub enum Hasher {
SHA256(Sha256),
SHA384(Sha384),
SHA512(Sha512),
@@ -109,7 +109,7 @@ enum Hasher {
impl Hasher {
// update hash value with new data
- fn update(&mut self, data: &[u8]) {
+ pub fn update(&mut self, data: &[u8]) {
use Hasher::*;
// update the hash
match self {
@@ -120,7 +120,7 @@ impl Hasher {
}
// comsume hasher and return the final digest
- fn finalize(hasher_enum: Hasher) -> Vec<u8> {
+ pub fn finalize(hasher_enum: Hasher) -> Vec<u8> {
use Hasher::*;
// return the hash
match hasher_enum {
@@ -132,27 +132,82 @@ impl Hasher {
}
// Return hash bytes for desired hashing algorithm.
-pub fn hash_by_alg(alg: &str, data: &[u8], exclusions: Option<Vec<Exclusion>>) -> Vec<u8> {
+pub fn hash_by_alg(alg: &str, data: &[u8], exclusions: Option<Vec<HashRange>>) -> Vec<u8> {
let mut reader = Cursor::new(data);
- hash_stream_by_alg(alg, &mut reader, exclusions).unwrap_or(Vec::new())
+ hash_stream_by_alg(alg, &mut reader, exclusions, true).unwrap_or(Vec::new())
+}
+
+// Return hash inclusive bytes for desired hashing algorithm.
+pub fn hash_by_alg_with_inclusions(alg: &str, data: &[u8], inclusions: Vec<HashRange>) -> Vec<u8> {
+ let mut reader = Cursor::new(data);
+
+ hash_stream_by_alg(alg, &mut reader, Some(inclusions), false).unwrap_or(Vec::new())
}
// Return hash bytes for asset using desired hashing algorithm.
pub fn hash_asset_by_alg(
alg: &str,
asset_path: &Path,
- exclusions: Option<Vec<Exclusion>>,
+ exclusions: Option<Vec<HashRange>>,
) -> Result<Vec<u8>> {
let mut file = File::open(asset_path)?;
- hash_stream_by_alg(alg, &mut file, exclusions)
+ hash_stream_by_alg(alg, &mut file, exclusions, true)
}
-// Return hash bytes for stream using desired hashing algorithm.
+// Return hash inclusive bytes for asset using desired hashing algorithm.
+pub fn hash_asset_by_alg_with_inclusions(
+ alg: &str,
+ asset_path: &Path,
+ inclusions: Vec<HashRange>,
+) -> Result<Vec<u8>> {
+ let mut file = File::open(asset_path)?;
+ hash_stream_by_alg(alg, &mut file, Some(inclusions), false)
+}
+
+/* Returns hash bytes for a stream using desired hashing algorithm. The function handles the many
+ possible hash requirements of C2PA. The function accepts a source stream 'data', an optional
+ set of hash ranges 'hash_range' and a boolean to indicate whether the hash range is an exclusion
+ or inclusion set of hash ranges.
+
+ The basic case is to hash a stream without hash ranges:
+ The data represents a single contiguous stream of bytes to be hash where D are data bytes
+
+ to_be_hashed: [DDDDDDDDD...DDDDDDDDDD]
+
+ The data is then chunked and hashed in groups to reduce memory
+ footprint and increase performance.
+
+ The most common case for C2PA is the use of an exclusion hash. In this case the 'hash_range' indicate
+ which byte ranges should be excluded shown here depicted with I for included bytes and X for excluded bytes
+
+ to_be_hashed: [IIIIXXXIIIIXXXXXIIIXXIII...IIII]
+
+ In this case the data is split into a set of ranges covering the included bytes. The set of ranged bytes
+ are then chunked and hashed just like the default case.
+
+ The opposite of this is when 'is_exclusion' is set to false indicating the 'hash_ranges' represent the bytes
+ to include in the hash. Here are the bytes in 'data' are excluded except those explicitly referenced.
+
+ to_be_hashed: [XXXXXXIIIIXXXXXIIXXXX...XXXX]
+
+ Again a set of ranged bytes are created and hashed as described above.
+
+ The last case is a special requirement for BMFF based assets (exclusion hashes only). For this case we not
+ only hash the data but also the location where the data was found in the asset. To do this we add a special
+ HashRange object to the hash ranges to indicate which locations in the stream require this special offset
+ hash. To make processing efficient we again split the data into ranges at not just the exclusion
+ points but also for these markers. The hashing loop knows to pause at these special marker ranges to insert
+ the hash of the offset. The stream sent to the hashing loop logically looks like this where M is the marker.
+ to_be_hashed: [IIIIIXXXXXMIIIIIMXXXXXMXXXXIII...III]
+
+ The data is again split into range sets breaking at the exclusion points and now also the markers.
+*/
pub fn hash_stream_by_alg<R>(
alg: &str,
data: &mut R,
- exclusions: Option<Vec<Exclusion>>,
+ hash_range: Option<Vec<HashRange>>,
+ is_exclusion: bool,
) -> Result<Vec<u8>>
where
R: Read + Seek + ?Sized,
@@ -176,58 +231,85 @@ where
let data_len = data.seek(SeekFrom::End(0))?;
data.rewind()?;
- let ranges = match exclusions {
- Some(mut e) if !e.is_empty() => {
+ let ranges = match hash_range {
+ Some(mut hr) if !hr.is_empty() => {
// hash data skipping excluded regions
// sort the exclusions
- e.sort_by_key(|a| a.start());
+ hr.sort_by_key(|a| a.start());
// verify structure of blocks
- let num_blocks = e.len();
- let exclusion_end = e[num_blocks - 1].start() + e[num_blocks - 1].length();
+ let num_blocks = hr.len();
+ let range_end = hr[num_blocks - 1].start() + hr[num_blocks - 1].length();
let data_end = data_len - 1;
- // if not enough range we will just calc to the end
- if data_len < exclusion_end as u64 {
+ // range extends past end of file so fail
+ if data_len < range_end as u64 {
return Err(Error::BadParam(
"The exclusion range exceed the data length".to_string(),
));
}
- //build final ranges
- let mut ranges_vec: Vec<RangeInclusive<u64>> = Vec::new();
- let mut ranges = RangeSet::<[RangeInclusive<u64>; 1]>::from(0..=data_end);
- for exclusion in e {
- let end = (exclusion.start() + exclusion.length() - 1) as u64;
- let exclusion_start = exclusion.start() as u64;
- ranges.remove_range(exclusion_start..=end);
-
- // add new BMFF V2 offset as a new range to be included so that we can
- // pause to add the offset hash
- if let Some(offset) = exclusion.bmff_offset() {
- ranges_vec.push(RangeInclusive::new(offset, offset));
- bmff_v2_starts.push(offset);
+ if is_exclusion {
+ //build final ranges
+ let mut ranges_vec: Vec<RangeInclusive<u64>> = Vec::new();
+ let mut ranges = RangeSet::<[RangeInclusive<u64>; 1]>::from(0..=data_end);
+ for exclusion in hr {
+ let end = (exclusion.start() + exclusion.length() - 1) as u64;
+ let exclusion_start = exclusion.start() as u64;
+ ranges.remove_range(exclusion_start..=end);
+
+ // add new BMFF V2 offset as a new range to be included so that we can
+ // pause to add the offset hash
+ if let Some(offset) = exclusion.bmff_offset() {
+ bmff_v2_starts.push(offset);
+ }
}
- }
- // merge standard ranges and BMFF V2 ranges into single list
- if !bmff_v2_starts.is_empty() {
- // add regularly included ranges
- for r in ranges.into_smallvec() {
- ranges_vec.push(r);
+ // merge standard ranges and BMFF V2 ranges into single list
+ if !bmff_v2_starts.is_empty() {
+ // remove any offset hashes that would be excluded
+ bmff_v2_starts.retain(|o| ranges.iter().any(|r| *o + 1 == r));
+
+ // add in remaining BMFF V2 offsets
+ for os in bmff_v2_starts.iter() {
+ ranges_vec.push(RangeInclusive::new(*os, *os));
+ }
+
+ // add regularly included ranges
+ for r in ranges.into_smallvec() {
+ ranges_vec.push(r);
+ }
+
+ // sort by start position
+ ranges_vec.sort_by(|a, b| {
+ let a_start = a.start();
+ let b_start = b.start();
+ a_start.cmp(b_start)
+ });
+
+ ranges_vec
+ } else {
+ for r in ranges.into_smallvec() {
+ ranges_vec.push(r);
+ }
+ ranges_vec
}
-
- // sort by start position
- ranges_vec.sort_by(|a, b| {
- let a_start = a.start();
- let b_start = b.start();
- a_start.cmp(b_start)
- });
-
- ranges_vec
} else {
- for r in ranges.into_smallvec() {
- ranges_vec.push(r);
+ //build final ranges
+ let mut ranges_vec: Vec<RangeInclusive<u64>> = Vec::new();
+ for inclusion in hr {
+ let end = (inclusion.start() + inclusion.length() - 1) as u64;
+ let inclusion_start = inclusion.start() as u64;
+
+ // add new BMFF V2 offset as a new range to be included so that we can
+ // pause to add the offset hash
+ if let Some(offset) = inclusion.bmff_offset() {
+ ranges_vec.push(RangeInclusive::new(offset, offset));
+ bmff_v2_starts.push(offset);
+ }
+
+ // add inclusion
+ ranges_vec.push(RangeInclusive::new(inclusion_start, end));
}
ranges_vec
}
@@ -329,7 +411,7 @@ pub fn verify_by_alg(
alg: &str,
hash: &[u8],
data: &[u8],
- exclusions: Option<Vec<Exclusion>>,
+ exclusions: Option<Vec<HashRange>>,
) -> bool {
// hash with the same algorithm as target
let data_hash = hash_by_alg(alg, data, exclusions);
@@ -341,7 +423,7 @@ pub fn verify_asset_by_alg(
alg: &str,
hash: &[u8],
asset_path: &Path,
- exclusions: Option<Vec<Exclusion>>,
+ exclusions: Option<Vec<HashRange>>,
) -> bool {
// hash with the same algorithm as target
if let Ok(data_hash) = hash_asset_by_alg(alg, asset_path, exclusions) {
@@ -350,6 +432,24 @@ pub fn verify_asset_by_alg(
false
}
}
+
+pub fn verify_stream_by_alg<R>(
+ alg: &str,
+ hash: &[u8],
+ reader: &mut R,
+ hash_range: Option<Vec<HashRange>>,
+ is_exclusion: bool,
+) -> bool
+where
+ R: Read + Seek + ?Sized,
+{
+ if let Ok(data_hash) = hash_stream_by_alg(alg, reader, hash_range, is_exclusion) {
+ vec_compare(hash, &data_hash)
+ } else {
+ false
+ }
+}
+
/// Return a multihash (Sha256) of array of bytes
#[allow(dead_code)]
pub fn hash256(data: &[u8]) -> String {
@@ -485,3 +585,14 @@ pub fn hash_as_source(in_hash: &str, data: &[u8]) -> Option<String> {
Err(_) => None,
}
}
+
+// Used by Merkle tree calculations to generate the pair wise hash
+pub fn concat_and_hash(alg: &str, left: &[u8], right: Option<&[u8]>) -> Vec<u8> {
+ let mut temp = left.to_vec();
+
+ if let Some(r) = right {
+ temp.append(&mut r.to_vec())
+ }
+
+ hash_by_alg(alg, &temp, None)
+}
diff --git a/sdk/src/utils/merkle.rs b/sdk/src/utils/merkle.rs
@@ -0,0 +1,150 @@
+// Copyright 2023 Adobe. All rights reserved.
+// This file is licensed to you under the Apache License,
+// Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
+// or the MIT license (http://opensource.org/licenses/MIT),
+// at your option.
+
+// Unless required by applicable law or agreed to in writing,
+// this software is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR REPRESENTATIONS OF ANY KIND, either express or
+// implied. See the LICENSE-MIT and LICENSE-APACHE files for the
+// specific language governing permissions and limitations under
+// each license.
+
+use extfmt::Hexlify;
+
+use super::hash_utils::{concat_and_hash, hash_by_alg};
+use crate::{Error, Result};
+
+#[derive(Default, Clone, PartialEq, Debug)]
+pub struct MerkleNode(pub Vec<u8>);
+
+// Implements Merkle tree support corresponding to the C2PA spec variant. The Merkle tree is not reduced and
+// all leaves live at the bottom most level. If the last layer node is an odd index (lacking a matching pair),
+// its node value is propagated to parent layer, no cloning or hashing is expected. Null tree entries do not contribute to the hashes.
+pub struct C2PAMerkleTree {
+ pub leaves: Vec<MerkleNode>,
+ pub layers: Vec<Vec<MerkleNode>>,
+}
+
+#[allow(dead_code)]
+impl C2PAMerkleTree {
+ pub fn from_leaves(leaves: Vec<MerkleNode>, alg: &str, hash_leaves: bool) -> C2PAMerkleTree {
+ let leaves = if hash_leaves {
+ leaves
+ .into_iter()
+ .map(|leaf| {
+ let hash = hash_by_alg(alg, &leaf.0, None);
+ MerkleNode(hash)
+ })
+ .collect()
+ } else {
+ leaves // this handles the case when the leaves are already hashed
+ };
+
+ let layers = C2PAMerkleTree::generate_tree(alg, &leaves);
+
+ C2PAMerkleTree { leaves, layers }
+ }
+
+ // generate layer layout
+ pub fn to_layout(num_leaves: usize) -> Vec<usize> {
+ let mut layers = Vec::new();
+
+ layers.push(num_leaves);
+ let mut current_layer = layers[0];
+
+ while current_layer > 1 {
+ let parent_layer_index = layers.len();
+ let mut parent_layer_cnt: usize = 0;
+
+ for i in (0..current_layer).step_by(2) {
+ if i + 1 == current_layer {
+ parent_layer_cnt += 1;
+ continue;
+ }
+
+ parent_layer_cnt += 1;
+ }
+ layers.push(parent_layer_cnt);
+ current_layer = layers[parent_layer_index];
+ }
+
+ layers
+ }
+
+ pub fn get_root(&self) -> Option<&Vec<u8>> {
+ Some(&self.layers.last()?.first()?.0)
+ }
+
+ fn generate_tree(alg: &str, leaves: &[MerkleNode]) -> Vec<Vec<MerkleNode>> {
+ let mut layers = Vec::new();
+ layers.push(leaves.to_vec()); // set layer 0
+ let mut current_layer = &layers[0];
+
+ while current_layer.len() > 1 {
+ let parent_layer_index = layers.len();
+ let mut parent_layer = Vec::new();
+
+ for i in (0..current_layer.len()).step_by(2) {
+ if i + 1 == current_layer.len() {
+ // just pass the current hash since last node is unbalanced
+ parent_layer.push(MerkleNode(current_layer[i].0.clone()));
+ continue;
+ }
+ let left = ¤t_layer[i];
+ let right = if i + 1 == current_layer.len() {
+ left
+ } else {
+ ¤t_layer[i + 1]
+ };
+
+ parent_layer.push(MerkleNode(concat_and_hash(alg, &left.0, Some(&right.0))));
+ }
+ layers.push(parent_layer);
+ current_layer = &layers[parent_layer_index];
+ }
+ layers
+ }
+
+ pub fn get_proof_by_index(&self, leaf_indx: usize) -> Result<Vec<Vec<u8>>> {
+ if self.leaves.is_empty() || leaf_indx >= self.leaves.len() {
+ return Err(Error::BadParam(
+ "Merkle proof index out of range".to_string(),
+ ));
+ }
+
+ let mut proof: Vec<Vec<u8>> = Vec::new();
+ let mut index = leaf_indx;
+
+ for i in 0..self.layers.len() {
+ let layer = &self.layers[i];
+ let is_right = index % 2 == 1;
+
+ if is_right {
+ if index - 1 < layer.len() {
+ proof.push(layer[index - 1].0.clone());
+ }
+ } else if index + 1 < layer.len() {
+ proof.push(layer[index + 1].0.clone());
+ }
+ index /= 2;
+ }
+ Ok(proof)
+ }
+
+ pub fn num_layers_required(n: u32) -> i32 {
+ let f = 1.0 * n as f32;
+
+ f.log2().ceil() as i32
+ }
+
+ pub fn tree_dump(&self) {
+ for (i, layer) in self.layers.iter().enumerate() {
+ println!("Level: {i}");
+ for (j, mn) in layer.iter().enumerate() {
+ println!("{} (Node: {j})", Hexlify(&mn.0));
+ }
+ }
+ }
+}
diff --git a/sdk/src/utils/mod.rs b/sdk/src/utils/mod.rs
@@ -14,6 +14,7 @@
pub(crate) mod cbor_types;
#[allow(dead_code)]
pub(crate) mod hash_utils;
+pub(crate) mod merkle;
#[allow(dead_code)] // for wasm build
pub(crate) mod patch;
#[cfg(all(feature = "add_thumbnails", any(feature = "file_io")))]
