c2pa-rs

A fork of https://github.com/contentauth/c2pa-rs/
git clone git://archive.git.mtrnord.blog/mtrnords-photography-manager/c2pa-rs.git
Log | Files | Refs | README

commit 64f6ee5f2c520fce03f00314f1b4a8dd684b8333
parent 6e1af5771d8f7ce979b5206c06f8ffb9fd1a01eb
Author: mauricefisher64 <92736594+mauricefisher64@users.noreply.github.com>
Date:   Tue, 12 Jul 2022 07:42:16 -0400

Refactor code to limit memory usage and remove data copies during hash generation (#67)

* Refactor code to limit memory usage and remove data copies

* Clippy fixes

* more clippy fixes

* Use sentence case and capitalized acronyms in error messages

* Use sentence case in error message

Co-authored-by: Eric Scouten <scouten@adobe.com>
Diffstat:
Msdk/src/assertions/bmff_hash.rs | 43+++++++++++++++++++++++++++++++------------
Msdk/src/assertions/data_hash.rs | 45++++++++++++++++++++++++++++++---------------
Msdk/src/asset_handlers/bmff_io.rs | 22+++++++++++++++++-----
Msdk/src/asset_handlers/png_io.rs | 3+--
Msdk/src/claim.rs | 50++++++++++++++++++++++++++++++++++++++++----------
Msdk/src/store.rs | 67++++++++++++++++++++++++++++++++++++++++++++++++-------------------
Msdk/src/utils/cbor_types.rs | 6+++---
Msdk/src/utils/hash_utils.rs | 129+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----------
8 files changed, 284 insertions(+), 81 deletions(-)

diff --git a/sdk/src/assertions/bmff_hash.rs b/sdk/src/assertions/bmff_hash.rs @@ -25,8 +25,8 @@ use crate::{ assertions::labels, asset_handlers::bmff_io::bmff_to_jumbf_exclusions, cbor_types::UriT, - error::{wrap_io_err, Result}, - utils::hash_utils::{hash_by_alg, verify_by_alg}, + error::Result, + utils::hash_utils::{hash_asset_by_alg, verify_asset_by_alg, verify_by_alg}, Error, }; @@ -191,9 +191,6 @@ impl BmffHash { )); } - let mut data = fs::read(asset_path).map_err(wrap_io_err)?; - let mut data_reader = Cursor::new(data); - let alg = match self.alg { Some(ref a) => a.clone(), None => "sha256".to_string(), @@ -202,10 +199,10 @@ impl BmffHash { let bmff_exclusions = &self.exclusions; // convert BMFF exclusion map to flat exclusion list - let exclusions = bmff_to_jumbf_exclusions(&mut data_reader, bmff_exclusions)?; + let mut data = fs::File::open(asset_path)?; + let exclusions = bmff_to_jumbf_exclusions(&mut data, bmff_exclusions)?; - data = data_reader.into_inner(); // back to buffer - let hash = hash_by_alg(&alg, &data, Some(exclusions)); + let hash = hash_asset_by_alg(&alg, asset_path, Some(exclusions))?; if hash.is_empty() { Err(Error::BadParam("could not generate data hash".to_string())) @@ -215,10 +212,10 @@ impl BmffHash { } pub fn verify_in_memory_hash(&self, data: &[u8], alg: Option<String>) -> Result<()> { - let curr_alg = match alg { - Some(a) => a, - None => match self.alg { - Some(ref a) => a.clone(), + let curr_alg = match &self.alg { + Some(a) => a.clone(), + None => match alg { + Some(a) => a, None => "sha256".to_string(), }, }; @@ -236,6 +233,28 @@ impl BmffHash { Err(Error::HashMismatch("Hashes do not match".to_owned())) } } + + pub fn verify_hash(&self, asset_path: &Path, alg: Option<String>) -> Result<()> { + let curr_alg = match &self.alg { + Some(a) => a.clone(), + None => match alg { + Some(a) => a, + None => "sha256".to_string(), + }, + }; + + let bmff_exclusions = &self.exclusions; + + // convert BMFF exclusion map to flat exclusion list + let mut data = fs::File::open(asset_path)?; + let exclusions = bmff_to_jumbf_exclusions(&mut data, bmff_exclusions)?; + + if verify_asset_by_alg(&curr_alg, &self.hash, asset_path, Some(exclusions)) { + Ok(()) + } else { + Err(Error::HashMismatch("Hashes do not match".to_owned())) + } + } } impl AssertionCbor for BmffHash {} diff --git a/sdk/src/assertions/data_hash.rs b/sdk/src/assertions/data_hash.rs @@ -11,7 +11,7 @@ // specific language governing permissions and limitations under // each license. -use std::{fs, path::*}; +use std::path::*; use serde::{Deserialize, Serialize}; use serde_bytes::ByteBuf; @@ -20,8 +20,8 @@ use crate::{ assertion::{Assertion, AssertionBase, AssertionCbor}, assertions::labels, cbor_types::UriT, - error::{wrap_io_err, Error, Result}, - utils::hash_utils::{hash_by_alg, verify_by_alg, Exclusion}, + error::{Error, Result}, + utils::hash_utils::{hash_asset_by_alg, verify_asset_by_alg, verify_by_alg, Exclusion}, }; const ASSERTION_CREATION_VERSION: usize = 1; @@ -149,8 +149,6 @@ impl DataHash { )); } - let data = fs::read(asset_path).map_err(wrap_io_err)?; - let alg = match self.alg { Some(ref a) => a.clone(), None => "sha256".to_string(), @@ -158,8 +156,8 @@ impl DataHash { // sort the exclusions let hash = match self.exclusions { - Some(ref e) => hash_by_alg(&alg, &data, Some(e.clone())), - None => hash_by_alg(&alg, &data, None), + Some(ref e) => hash_asset_by_alg(&alg, asset_path, Some(e.clone()))?, + None => hash_asset_by_alg(&alg, asset_path, None)?, }; if hash.is_empty() { @@ -175,10 +173,10 @@ impl DataHash { return Err(Error::BadParam("asset hash is remote".to_owned())); } - let curr_alg = match alg { - Some(a) => a, - None => match self.alg { - Some(ref a) => a.clone(), + let curr_alg = match &self.alg { + Some(a) => a.clone(), + None => match alg { + Some(a) => a, None => "sha256".to_string(), }, }; @@ -194,9 +192,26 @@ impl DataHash { /// Used to verify a DataHash against an asset. #[allow(dead_code)] // used in tests - pub fn verify_hash(&self, asset_path: &Path) -> Result<()> { - let buf = fs::read(asset_path).map_err(wrap_io_err)?; - self.verify_in_memory_hash(&buf, self.alg.clone()) + pub fn verify_hash(&self, asset_path: &Path, alg: Option<String>) -> Result<()> { + if self.is_remote_hash() { + return Err(Error::BadParam("asset hash is remote".to_owned())); + } + + let curr_alg = match &self.alg { + Some(a) => a.clone(), + None => match alg { + Some(a) => a, + None => "sha256".to_string(), + }, + }; + + let exclusions = self.exclusions.as_ref().cloned(); + + if verify_asset_by_alg(&curr_alg, &self.hash, asset_path, exclusions) { + Ok(()) + } else { + Err(Error::HashMismatch("Hashes do not match".to_owned())) + } } /// Create a new instance from Assertion @@ -293,7 +308,7 @@ pub mod tests { data_hash.gen_hash(&ap).unwrap(); // verify - data_hash.verify_hash(&ap).unwrap(); + data_hash.verify_hash(&ap, None).unwrap(); let assertion = data_hash.to_assertion().unwrap(); diff --git a/sdk/src/asset_handlers/bmff_io.rs b/sdk/src/asset_handlers/bmff_io.rs @@ -12,7 +12,7 @@ // each license. use std::collections::HashMap; -use std::convert::From; +use std::convert::{From, TryFrom}; use std::fs::File; use std::io::{Read, Seek, SeekFrom, Write}; use std::path::Path; @@ -529,9 +529,15 @@ fn adjust_stco_and_co64<W: Write + CAIRead>( for _e in 0..entry_count { let offset = output.read_u32::<BigEndian>()?; let new_offset = if adjust < 0 { - offset - adjust.abs() as u32 + offset + - u32::try_from(adjust.abs()).map_err(|_| { + Error::BadParam("Bad BMFF offset adjustment".to_string()) + })? } else { - offset + adjust as u32 + offset + + u32::try_from(adjust).map_err(|_| { + Error::BadParam("Bad BMFF offset adjustment".to_string()) + })? }; entries.push(new_offset); } @@ -574,9 +580,15 @@ fn adjust_stco_and_co64<W: Write + CAIRead>( for _e in 0..entry_count { let offset = output.read_u64::<BigEndian>()?; let new_offset = if adjust < 0 { - offset - adjust.abs() as u64 + offset + - u64::try_from(adjust.abs()).map_err(|_| { + Error::BadParam("Bad BMFF offset adjustment".to_string()) + })? } else { - offset + adjust as u64 + offset + + u64::try_from(adjust).map_err(|_| { + Error::BadParam("Bad BMFF offset adjustment".to_string()) + })? }; entries.push(new_offset); } diff --git a/sdk/src/asset_handlers/png_io.rs b/sdk/src/asset_handlers/png_io.rs @@ -78,8 +78,7 @@ fn get_png_chunk_positions(f: &mut dyn CAIRead) -> Result<Vec<PngChunkPos>> { .map_err(|_err| Error::BadParam("PNG out of range".to_string()))?; // read crc - let _crc = f - .read_exact(&mut buf4) + f.read_exact(&mut buf4) .map_err(|_err| Error::BadParam("PNG out of range".to_string()))?; let chunk_name = String::from_utf8(name.to_vec()) diff --git a/sdk/src/claim.rs b/sdk/src/claim.rs @@ -16,6 +16,7 @@ use serde::{Deserialize, Serialize}; use serde_json::{json, Map, Value}; use std::collections::HashMap; use std::fmt; +use std::path::Path; use uuid::Uuid; use crate::assertion::{ @@ -45,6 +46,11 @@ use HashedUri as C2PAAssertion; const GH_FULL_VERSION_LIST: &str = "Sec-CH-UA-Full-Version-List"; const GH_UA: &str = "Sec-CH-UA"; +pub enum ClaimAssetData<'a> { + PathData(&'a Path), + ByteData(&'a [u8]), +} + #[derive(PartialEq, Clone)] // helper struct to allow arbitrary order for assertions stored in jumbf. The instance is // stored separate from the Assertion to allow for late binding to the label. Also, @@ -710,9 +716,9 @@ impl Claim { /// Verify claim signature, assertion store and asset hashes /// claim - claim to be verified /// asset_bytes - reference to bytes of the asset - pub async fn verify_claim_async( + pub async fn verify_claim_async<'a>( claim: &Claim, - asset_bytes: &[u8], + asset_bytes: &'a [u8], is_provenance: bool, validation_log: &mut impl StatusTracker, ) -> Result<()> { @@ -750,15 +756,21 @@ impl Claim { validation_log, ) .await; - Claim::verify_internal(claim, asset_bytes, is_provenance, verified, validation_log) + Claim::verify_internal( + claim, + &ClaimAssetData::ByteData(asset_bytes), + is_provenance, + verified, + validation_log, + ) } /// Verify claim signature, assertion store and asset hashes /// claim - claim to be verified /// asset_bytes - reference to bytes of the asset - pub fn verify_claim( + pub fn verify_claim<'a>( claim: &Claim, - asset_bytes: &[u8], + asset_data: &ClaimAssetData<'a>, is_provenance: bool, validation_log: &mut impl StatusTracker, ) -> Result<()> { @@ -790,12 +802,12 @@ impl Claim { validation_log, ); - Claim::verify_internal(claim, asset_bytes, is_provenance, verified, validation_log) + Claim::verify_internal(claim, asset_data, is_provenance, verified, validation_log) } - fn verify_internal( + fn verify_internal<'a>( claim: &Claim, - asset_bytes: &[u8], + asset_data: &ClaimAssetData<'a>, is_provenance: bool, verified: Result<ValidationInfo>, validation_log: &mut impl StatusTracker, @@ -962,7 +974,16 @@ impl Claim { let name = dh.name.as_ref().map_or(UNNAMED.to_string(), default_str); if !dh.is_remote_hash() { // only verify local hashes here - match dh.verify_in_memory_hash(asset_bytes, Some(claim.alg().to_string())) { + let hash_result = match asset_data { + ClaimAssetData::PathData(asset_path) => { + dh.verify_hash(asset_path, Some(claim.alg().to_string())) + } + ClaimAssetData::ByteData(asset_bytes) => { + dh.verify_in_memory_hash(asset_bytes, Some(claim.alg().to_string())) + } + }; + + match hash_result { Ok(_a) => { let log_item = log_item!( claim.assertion_uri(&dh_assertion.label()), @@ -996,7 +1017,16 @@ impl Claim { let name = dh.name().map_or("unnamed".to_string(), default_str); - match dh.verify_in_memory_hash(asset_bytes, Some(claim.alg().to_string())) { + let hash_result = match asset_data { + ClaimAssetData::PathData(asset_path) => { + dh.verify_hash(asset_path, Some(claim.alg().to_string())) + } + ClaimAssetData::ByteData(asset_bytes) => { + dh.verify_in_memory_hash(asset_bytes, Some(claim.alg().to_string())) + } + }; + + match hash_result { Ok(_a) => { let log_item = log_item!( claim.assertion_uri(&dh_assertion.label()), diff --git a/sdk/src/store.rs b/sdk/src/store.rs @@ -14,7 +14,7 @@ use crate::{ assertion::{Assertion, AssertionBase, AssertionDecodeError, AssertionDecodeErrorCause}, assertions::{labels, Ingredient, Relationship}, - claim::{Claim, ClaimAssertion}, + claim::{Claim, ClaimAssertion, ClaimAssetData}, error::{Error, Result}, hash_utils::{hash_by_alg, vec_compare, verify_by_alg}, jumbf::{self, boxes::*}, @@ -980,10 +980,10 @@ impl Store { } // wake the ingredients and validate - fn ingredient_checks( + fn ingredient_checks<'a>( store: &Store, claim: &Claim, - asset_bytes: &[u8], + asset_data: &ClaimAssetData<'a>, validation_log: &mut impl StatusTracker, ) -> Result<()> { let mut num_parent_ofs = 0; @@ -1026,7 +1026,7 @@ impl Store { // make sure // verify the ingredient claim - Claim::verify_claim(ingredient, asset_bytes, false, validation_log)?; + Claim::verify_claim(ingredient, asset_data, false, validation_log)?; } else { let log_item = log_item!( &c2pa_manifest.url(), @@ -1181,18 +1181,18 @@ impl Store { /// xmp_str: String containing entire XMP block of the asset /// asset_bytes: bytes of the asset to be verified /// validation_log: If present all found errors are logged and returned, other wise first error causes exit and is returned - pub fn verify_store( + pub fn verify_store<'a>( store: &Store, xmp_opt: Option<String>, - asset_bytes: &[u8], + asset_data: &ClaimAssetData<'a>, validation_log: &mut impl StatusTracker, ) -> Result<()> { let claim = Store::provenance_checks(store, xmp_opt, validation_log)?; // verify the provenance claim - Claim::verify_claim(claim, asset_bytes, true, validation_log)?; + Claim::verify_claim(claim, asset_data, true, validation_log)?; - Store::ingredient_checks(store, claim, asset_bytes, validation_log)?; + Store::ingredient_checks(store, claim, asset_data, validation_log)?; Ok(()) } @@ -1549,17 +1549,38 @@ impl Store { /// asset_path: path to input asset /// validation_log: If present all found errors are logged and returned, otherwise first error causes exit and is returned #[cfg(feature = "file_io")] - pub fn verify_from_path( + pub fn verify_from_path<'a>( &mut self, - asset_path: &Path, + asset_path: &'a Path, validation_log: &mut impl StatusTracker, ) -> Result<()> { let ext = get_supported_file_extension(asset_path).ok_or(Error::UnsupportedType)?; - // load the bytes - let buf = fs::read(asset_path).map_err(crate::error::wrap_io_err)?; + let cai_loader = get_cailoader_handler(&ext).ok_or(Error::UnsupportedType)?; + + let mut asset_reader = fs::File::open(asset_path)?; + + // read xmp if available + let xmp_opt = cai_loader.read_xmp(&mut asset_reader); - self.verify_from_buffer(&buf, &ext, validation_log) + let xmp_copy = xmp_opt.clone(); + + Store::verify_store( + self, + xmp_opt, + &ClaimAssetData::PathData(asset_path), + validation_log, + )?; + + // set the provenance if there is xmp otherwise it will default to active manifest + if let Some(xmp) = xmp_copy { + if let Some(xmp_provenance) = extract_provenance(&xmp) { + let claim_label = Store::manifest_label_from_path(&xmp_provenance); + self.set_provenance_path(&claim_label); + } + } + + Ok(()) } // verify from a buffer without file i/o @@ -1580,7 +1601,12 @@ impl Store { let buf = buf_reader.into_inner(); - Store::verify_store(self, xmp_opt, buf, validation_log)?; + Store::verify_store( + self, + xmp_opt, + &ClaimAssetData::ByteData(buf), + validation_log, + )?; // set the provenance if there is xmp otherwise it will default to active manifest if let Some(xmp) = xmp_copy { @@ -1695,22 +1721,25 @@ impl Store { /// data: reference to bytes of the the file /// verify: if true will run verification checks when loading /// validation_log: If present all found errors are logged and returned, otherwise first error causes exit and is returned - pub fn load_from_memory( + pub fn load_from_memory<'a>( asset_type: &str, - data: &[u8], + data: &'a [u8], verify: bool, validation_log: &mut impl StatusTracker, ) -> Result<Store> { Store::get_store_from_memory(asset_type, data, validation_log).and_then( |(mut store, xmp_opt)| { - let buf_reader = Cursor::new(data); - // verify the store if verify { let xmp_copy = xmp_opt.clone(); // verify store and claims - Store::verify_store(&store, xmp_opt, buf_reader.get_ref(), validation_log)?; + Store::verify_store( + &store, + xmp_opt, + &ClaimAssetData::ByteData(data), + validation_log, + )?; // set the provenance if checks pass & has xmp, otherwise default to active manifest if let Some(xmp) = xmp_copy { diff --git a/sdk/src/utils/cbor_types.rs b/sdk/src/utils/cbor_types.rs @@ -42,7 +42,7 @@ impl<'de> Deserialize<'de> for DateT { } } -impl<'a> AsRef<str> for DateT { +impl AsRef<str> for DateT { fn as_ref(&self) -> &str { &self.0 } @@ -74,7 +74,7 @@ impl<'de> Deserialize<'de> for UriT { } } -impl<'a> AsRef<str> for UriT { +impl AsRef<str> for UriT { fn as_ref(&self) -> &str { &self.0 } @@ -105,7 +105,7 @@ impl<'de> Deserialize<'de> for BytesT { } } -impl<'a> AsRef<Vec<u8>> for BytesT { +impl AsRef<Vec<u8>> for BytesT { fn as_ref(&self) -> &Vec<u8> { &self.0 } diff --git a/sdk/src/utils/hash_utils.rs b/sdk/src/utils/hash_utils.rs @@ -11,7 +11,12 @@ // specific language governing permissions and limitations under // each license. -use std::ops::RangeInclusive; +use std::{ + fs::File, + io::{Read, Seek, SeekFrom}, + ops::RangeInclusive, + path::Path, +}; use log::{debug, warn}; use serde::{Deserialize, Serialize}; @@ -25,6 +30,10 @@ use range_set::RangeSet; // direct sha functions use sha2::{Digest, Sha256, Sha384, Sha512}; +use crate::{Error, Result}; + +const MAX_HASH_BUF: usize = 1024 * 1024 * 1024; // cap memory usage to 1GB + #[derive(Clone, Serialize, Deserialize, Debug, PartialEq)] pub struct Exclusion { start: usize, @@ -106,7 +115,7 @@ impl Hasher { } } -// return hash bytes for desired hashing algoritm +// Return hash bytes for desired hashing algorithm. pub fn hash_by_alg(alg: &str, data: &[u8], exclusions: Option<Vec<Exclusion>>) -> Vec<u8> { use Hasher::*; let mut hasher_enum = match alg { @@ -123,16 +132,7 @@ pub fn hash_by_alg(alg: &str, data: &[u8], exclusions: Option<Vec<Exclusion>>) - }; match exclusions { - Some(mut e) => { - // hash all content - if e.is_empty() { - // add the data - hasher_enum.update(data); - - // return the hash - return Hasher::finalize(hasher_enum); - } - + Some(mut e) if !e.is_empty() => { // hash data skipping excluded regions // sort the exclusions e.sort_by_key(|a| a.start()); @@ -143,7 +143,7 @@ pub fn hash_by_alg(alg: &str, data: &[u8], exclusions: Option<Vec<Exclusion>>) - let data_len = data.len(); let data_end = data_len - 1; - // if not enough range we will just cacl to the end + // if not enough range we will just calc to the end if data_len < exclusion_end { debug!("the exclusion range exceed the data length"); return Vec::new(); @@ -164,7 +164,7 @@ pub fn hash_by_alg(alg: &str, data: &[u8], exclusions: Option<Vec<Exclusion>>) - // return the hash Hasher::finalize(hasher_enum) } - None => { + _ => { // add the data hasher_enum.update(data); @@ -174,7 +174,92 @@ pub fn hash_by_alg(alg: &str, data: &[u8], exclusions: Option<Vec<Exclusion>>) - } } -// verify the hash using the specifiied alogrithm +// Return hash bytes for assset using desired hashing algorithm. +pub fn hash_asset_by_alg( + alg: &str, + asset_path: &Path, + exclusions: Option<Vec<Exclusion>>, +) -> Result<Vec<u8>> { + use Hasher::*; + let mut hasher_enum = match alg { + "sha256" => SHA256(Sha256::new()), + "sha384" => SHA384(Sha384::new()), + "sha512" => SHA512(Sha512::new()), + _ => { + warn!( + "Unsupported hashing algorithm: {}, substituting sha256", + alg + ); + SHA256(Sha256::new()) + } + }; + + let mut data = File::open(asset_path)?; + let data_len = data.seek(SeekFrom::End(0))?; + data.seek(SeekFrom::Start(0))?; + + let ranges = match exclusions { + Some(mut e) if !e.is_empty() => { + // hash data skipping excluded regions + // sort the exclusions + e.sort_by_key(|a| a.start()); + + // verify structure of blocks + let num_blocks = e.len(); + let exclusion_end = e[num_blocks - 1].start() + e[num_blocks - 1].length(); + let data_end = data_len - 1; + + // if not enough range we will just calc to the end + if data_len < exclusion_end as u64 { + return Err(Error::BadParam( + "The exclusion range exceed the data length".to_string(), + )); + } + + //build final ranges + let mut ranges = RangeSet::<[RangeInclusive<u64>; 1]>::from(0..=data_end); + for exclusion in e { + let end = (exclusion.start() + exclusion.length() - 1) as u64; + let exclusion_start = exclusion.start() as u64; + ranges.remove_range(exclusion_start..=end); + } + + ranges + } + _ => { + let data_end = data_len - 1; + RangeSet::<[RangeInclusive<u64>; 1]>::from(0..=data_end) + } + }; + + // hash the data for ranges + for r in ranges.into_smallvec() { + let start = r.start(); + let end = r.end(); + let mut chunk_left = end - start + 1; + + // move to start of range + data.seek(SeekFrom::Start(*start))?; + + loop { + let mut chunk = vec![0u8; std::cmp::min(chunk_left as usize, MAX_HASH_BUF)]; + + data.read_exact(&mut chunk)?; + + hasher_enum.update(&chunk); + + chunk_left -= chunk.len() as u64; + if chunk_left == 0 { + break; + } + } + } + + // return the hash + Ok(Hasher::finalize(hasher_enum)) +} + +// verify the hash using the specified alogrithm pub fn verify_by_alg( alg: &str, hash: &[u8], @@ -186,6 +271,20 @@ pub fn verify_by_alg( vec_compare(hash, &data_hash) } +// verify the hash using the specified alogrithm +pub fn verify_asset_by_alg( + alg: &str, + hash: &[u8], + asset_path: &Path, + exclusions: Option<Vec<Exclusion>>, +) -> bool { + // hash with the same algorithm as target + if let Ok(data_hash) = hash_asset_by_alg(alg, asset_path, exclusions) { + vec_compare(hash, &data_hash) + } else { + false + } +} /// Return a multihash (Sha256) of array of bytes #[allow(dead_code)] pub fn hash256(data: &[u8]) -> String {