commit 2cb845f1a9dc2a1faaebabb85eef78af09dc9800
parent 268e9e6b23bc0b25945bfc9e1e688bdc4cbcda37
Author: mauricefisher64 <92736594+mauricefisher64@users.noreply.github.com>
Date: Mon, 10 Jul 2023 21:54:41 -0400
(MINOR) Embeddable manifest support (#266)
* Initial box hash support
* Full box hash implementation for JPG
* Box hash fixes
* More unit tests and safety checks
* Add API to get embeddable manifest
* Add missing test file
* Change timestamp message imprint to include entire protected header
* Format fixes
* Fix for mixing cbor and json in a single object
* Simplify fix to use existing code
* Rename test file to independent of other uses.
* Initial support for new external hashing support
* New external hashing support
* Switch new functions to be async
* Allow users to set data hash
* Disable unit test that is not ready to be enabled
* Adds manifest SDK support for box and data hashes
manifest.data_hash_placeholder()
manifest.data_hash_embeddable_manifest
manifest.box_hash_embeddable_manifest
* added manifest box hash unit test
---------
Co-authored-by: Gavin Peacock <gpeacock@adobe.com>
Diffstat:
10 files changed, 817 insertions(+), 77 deletions(-)
diff --git a/sdk/src/asset_handlers/jpeg_io.rs b/sdk/src/asset_handlers/jpeg_io.rs
@@ -15,11 +15,11 @@ use std::{
collections::HashMap,
convert::{From, TryFrom},
fs::File,
- io::{BufReader, Cursor},
+ io::{BufReader, Cursor, Write},
path::*,
};
-use byteorder::{BigEndian, ReadBytesExt};
+use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
use img_parts::{
jpeg::{
markers::{self, P, RST0, RST7, Z},
@@ -33,8 +33,8 @@ use tempfile::Builder;
use crate::{
assertions::{BoxMap, C2PA_BOXHASH},
asset_io::{
- AssetBoxHash, AssetIO, CAIRead, CAIReadWrite, CAIReader, CAIWriter, HashBlockObjectType,
- HashObjectPositions, RemoteRefEmbed, RemoteRefEmbedType,
+ AssetBoxHash, AssetIO, CAIRead, CAIReadWrite, CAIReader, CAIWriter, ComposedManifestRef,
+ HashBlockObjectType, HashObjectPositions, RemoteRefEmbed, RemoteRefEmbedType,
},
error::{Error, Result},
};
@@ -541,6 +541,10 @@ impl AssetIO for JpegIO {
Some(self)
}
+ fn composed_data_ref(&self) -> Option<&dyn ComposedManifestRef> {
+ Some(self)
+ }
+
fn supported_types(&self) -> &[&str] {
&SUPPORTED_TYPES
}
@@ -832,10 +836,82 @@ impl AssetBoxHash for JpegIO {
}
}
+impl ComposedManifestRef for JpegIO {
+ fn compose_manifest(&self, manifest_data: &[u8], _format: &str) -> Result<Vec<u8>> {
+ let jumbf_len = manifest_data.len();
+ let num_segments = (jumbf_len / MAX_JPEG_MARKER_SIZE) + 1;
+ let mut seg_chucks = manifest_data.chunks(MAX_JPEG_MARKER_SIZE);
+
+ let mut segments = Vec::new();
+
+ for seg in 1..num_segments + 1 {
+ /*
+ If the size of the box payload is less than 2^32-8 bytes,
+ then all fields except the XLBox field, that is: Le, CI, En, Z, LBox and TBox,
+ shall be present in all JPEG XT marker segment representing this box,
+ regardless of whether the marker segments starts this box,
+ or continues a box started by a former JPEG XT Marker segment.
+ */
+ // we need to prefix the JUMBF with the JPEG XT markers (ISO 19566-5)
+ // CI: JPEG extensions marker - JP
+ // En: Box Instance Number - 0x0001
+ // (NOTE: can be any unique ID, so we pick one that shouldn't conflict)
+ // Z: Packet sequence number - 0x00000001...
+ let ci = vec![0x4a, 0x50];
+ let en = vec![0x02, 0x11];
+ let z: u32 = u32::try_from(seg)
+ .map_err(|_| Error::InvalidAsset("Too many JUMBF segments".to_string()))?; //seg.to_be_bytes();
+
+ let mut seg_data = Vec::new();
+ seg_data.extend(ci);
+ seg_data.extend(en);
+ seg_data.extend(z.to_be_bytes());
+ if seg > 1 {
+ // the LBox and TBox are already in the JUMBF
+ // but we need to duplicate them in all other segments
+ let lbox_tbox = &manifest_data[..8];
+ seg_data.extend(lbox_tbox);
+ }
+ if seg_chucks.len() > 0 {
+ // make sure we have some...
+ if let Some(next_seg) = seg_chucks.next() {
+ seg_data.extend(next_seg);
+ }
+ } else {
+ seg_data.extend(manifest_data);
+ }
+
+ let seg_bytes = Bytes::from(seg_data);
+ let app11_segment = JpegSegment::new_with_contents(markers::APP11, seg_bytes);
+ segments.push(app11_segment);
+ }
+
+ let output = Vec::with_capacity(manifest_data.len() * 2);
+ let mut out_stream = Cursor::new(output);
+
+ // right out segments
+ for s in segments {
+ // maker
+ out_stream.write_u8(markers::P)?;
+ out_stream.write_u8(s.marker())?;
+
+ //len
+ out_stream.write_u16::<BigEndian>(s.contents().len() as u16 + 2)?;
+
+ // data
+ out_stream.write_all(s.contents())?;
+ }
+
+ Ok(out_stream.into_inner())
+ }
+}
+
#[cfg(test)]
pub mod tests {
#![allow(clippy::unwrap_used)]
+ use std::io::{Read, Seek};
+
use img_parts::Bytes;
use super::*;
@@ -933,4 +1009,59 @@ pub mod tests {
assert!(read_xmp.contains(test_msg));
}
+
+ #[test]
+ fn test_embeddable_manifest() {
+ let jpeg_io = JpegIO {};
+
+ let source = crate::utils::test::fixture_path("CA.jpg");
+
+ let ol = jpeg_io.get_object_locations(&source).unwrap();
+
+ let cai_loc = ol
+ .iter()
+ .find(|o| o.htype == HashBlockObjectType::Cai)
+ .unwrap();
+ let curr_manifest = jpeg_io.read_cai_store(&source).unwrap();
+
+ let temp_dir = tempfile::tempdir().unwrap();
+ let output = crate::utils::test::temp_dir_path(&temp_dir, "CA_test.jpg");
+
+ std::fs::copy(source, &output).unwrap();
+
+ // remove existing
+ jpeg_io.remove_cai_store(&output).unwrap();
+
+ // generate new manifest data
+ let em = jpeg_io
+ .composed_data_ref()
+ .unwrap()
+ .compose_manifest(&curr_manifest, "jpeg")
+ .unwrap();
+
+ // insert new manifest
+ let outbuf = Vec::new();
+ let mut out_stream = Cursor::new(outbuf);
+
+ let mut before = vec![0u8; cai_loc.offset];
+ let mut in_file = std::fs::File::open(&output).unwrap();
+
+ // write before
+ in_file.read_exact(before.as_mut_slice()).unwrap();
+ out_stream.write_all(&before).unwrap();
+
+ // write composed bytes
+ out_stream.write_all(&em).unwrap();
+
+ // write bytes after
+ let mut after_buf = Vec::new();
+ in_file.read_to_end(&mut after_buf).unwrap();
+ out_stream.write_all(&after_buf).unwrap();
+
+ // read manifest back in from new in-memory JPEG
+ out_stream.rewind().unwrap();
+ let restored_manifest = jpeg_io.read_cai(&mut out_stream).unwrap();
+
+ assert_eq!(&curr_manifest, &restored_manifest);
+ }
}
diff --git a/sdk/src/asset_handlers/png_io.rs b/sdk/src/asset_handlers/png_io.rs
@@ -24,8 +24,8 @@ use serde_bytes::ByteBuf;
use crate::{
assertions::{BoxMap, C2PA_BOXHASH},
asset_io::{
- AssetBoxHash, AssetIO, CAIRead, CAIReadWrite, CAIReader, CAIWriter, HashBlockObjectType,
- HashObjectPositions, RemoteRefEmbed, RemoteRefEmbedType,
+ AssetBoxHash, AssetIO, CAIRead, CAIReadWrite, CAIReader, CAIWriter, ComposedManifestRef,
+ HashBlockObjectType, HashObjectPositions, RemoteRefEmbed, RemoteRefEmbedType,
},
error::{Error, Result},
};
@@ -565,6 +565,10 @@ impl AssetIO for PngIO {
Some(self)
}
+ fn composed_data_ref(&self) -> Option<&dyn ComposedManifestRef> {
+ Some(self)
+ }
+
fn supported_types(&self) -> &[&str] {
&SUPPORTED_TYPES
}
@@ -654,11 +658,33 @@ impl AssetBoxHash for PngIO {
}
}
+impl ComposedManifestRef for PngIO {
+ fn compose_manifest(&self, manifest_data: &[u8], _format: &str) -> Result<Vec<u8>> {
+ let mut cai_data = Vec::new();
+ let mut cai_encoder = png_pong::Encoder::new(&mut cai_data).into_chunk_enc();
+
+ // create CAI store chunk
+ let cai_unknown = png_pong::chunk::Unknown {
+ name: CAI_CHUNK,
+ data: manifest_data.to_vec(),
+ };
+
+ let mut cai_chunk = png_pong::chunk::Chunk::Unknown(cai_unknown);
+ cai_encoder
+ .encode(&mut cai_chunk)
+ .map_err(|_| Error::EmbeddingError)?;
+
+ Ok(cai_data)
+ }
+}
+
#[cfg(test)]
pub mod tests {
#![allow(clippy::panic)]
#![allow(clippy::unwrap_used)]
+ use std::io::Write;
+
use twoway::find_bytes;
use super::*;
@@ -850,4 +876,59 @@ pub mod tests {
_ => unreachable!(),
}
}
+
+ #[test]
+ fn test_embeddable_manifest() {
+ let png_io = PngIO {};
+
+ let source = crate::utils::test::fixture_path("exp-test1.png");
+
+ let ol = png_io.get_object_locations(&source).unwrap();
+
+ let cai_loc = ol
+ .iter()
+ .find(|o| o.htype == HashBlockObjectType::Cai)
+ .unwrap();
+ let curr_manifest = png_io.read_cai_store(&source).unwrap();
+
+ let temp_dir = tempfile::tempdir().unwrap();
+ let output = crate::utils::test::temp_dir_path(&temp_dir, "exp-test1-out.png");
+
+ std::fs::copy(source, &output).unwrap();
+
+ // remove existing
+ png_io.remove_cai_store(&output).unwrap();
+
+ // generate new manifest data
+ let em = png_io
+ .composed_data_ref()
+ .unwrap()
+ .compose_manifest(&curr_manifest, "png")
+ .unwrap();
+
+ // insert new manifest
+ let outbuf = Vec::new();
+ let mut out_stream = Cursor::new(outbuf);
+
+ let mut before = vec![0u8; cai_loc.offset];
+ let mut in_file = std::fs::File::open(&output).unwrap();
+
+ // write before
+ in_file.read_exact(before.as_mut_slice()).unwrap();
+ out_stream.write_all(&before).unwrap();
+
+ // write composed bytes
+ out_stream.write_all(&em).unwrap();
+
+ // write bytes after
+ let mut after_buf = Vec::new();
+ in_file.read_to_end(&mut after_buf).unwrap();
+ out_stream.write_all(&after_buf).unwrap();
+
+ // read manifest back in from new in-memory PNG
+ out_stream.rewind().unwrap();
+ let restored_manifest = png_io.read_cai(&mut out_stream).unwrap();
+
+ assert_eq!(&curr_manifest, &restored_manifest);
+ }
}
diff --git a/sdk/src/asset_handlers/tiff_io.rs b/sdk/src/asset_handlers/tiff_io.rs
@@ -26,8 +26,8 @@ use tempfile::Builder;
use crate::{
asset_io::{
- AssetIO, AssetPatch, CAIRead, CAIReadWrite, CAIReader, HashBlockObjectType,
- HashObjectPositions, RemoteRefEmbed, RemoteRefEmbedType,
+ AssetIO, AssetPatch, CAIRead, CAIReadWrite, CAIReader, ComposedManifestRef,
+ HashBlockObjectType, HashObjectPositions, RemoteRefEmbed, RemoteRefEmbedType,
},
error::{Error, Result},
};
@@ -1486,6 +1486,10 @@ impl AssetIO for TiffIO {
Some(self)
}
+ fn composed_data_ref(&self) -> Option<&dyn ComposedManifestRef> {
+ Some(self)
+ }
+
fn supported_types(&self) -> &[&str] {
&SUPPORTED_TYPES
}
@@ -1565,6 +1569,13 @@ impl RemoteRefEmbed for TiffIO {
}
}
+impl ComposedManifestRef for TiffIO {
+ // Return entire CAI block as Vec<u8>
+ fn compose_manifest(&self, manifest_data: &[u8], _format: &str) -> Result<Vec<u8>> {
+ Ok(manifest_data.to_vec())
+ }
+}
+
#[cfg(test)]
pub mod tests {
#![allow(clippy::panic)]
diff --git a/sdk/src/asset_io.rs b/sdk/src/asset_io.rs
@@ -188,10 +188,15 @@ pub trait AssetIO: Sync + Send {
None
}
- // Returns [`AssetBoxHah`] trait if this I/O handler supports box hashing.
+ // Returns [`AssetBoxHash`] trait if this I/O handler supports box hashing.
fn asset_box_hash_ref(&self) -> Option<&dyn AssetBoxHash> {
None
}
+
+ // Returns [`ComposedManifestRefEmbed`] trait if this I/O handler supports composed data.
+ fn composed_data_ref(&self) -> Option<&dyn ComposedManifestRef> {
+ None
+ }
}
// `AssetPatch` optimizes output generation for asset_io handlers that
@@ -239,3 +244,11 @@ pub trait RemoteRefEmbed {
embed_ref: RemoteRefEmbedType,
) -> Result<()>;
}
+
+/// `ComposedManifestRefEmbed` is used to generate a C2PA manifest. The
+/// returned `Vec<u8>` contains data preformatted to be directly compatible
+/// with the type specified in `format`.
+pub trait ComposedManifestRef {
+ // Return entire CAI block as Vec<u8>
+ fn compose_manifest(&self, manifest_data: &[u8], format: &str) -> Result<Vec<u8>>;
+}
diff --git a/sdk/src/manifest.rs b/sdk/src/manifest.rs
@@ -22,19 +22,21 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize};
use serde_json::Value;
use uuid::Uuid;
-#[cfg(feature = "file_io")]
-use crate::AsyncSigner;
use crate::{
assertion::{AssertionBase, AssertionData},
- assertions::{labels, Actions, CreativeWork, Exif, SoftwareAgent, Thumbnail, User, UserCbor},
+ assertions::{
+ labels, Actions, CreativeWork, DataHash, Exif, SoftwareAgent, Thumbnail, User, UserCbor,
+ },
asset_io::CAIRead,
claim::{Claim, RemoteManifest},
error::{Error, Result},
+ hash_utils::HashRange,
jumbf,
resource_store::{skip_serializing_resources, ResourceRef, ResourceStore},
salt::DefaultSalt,
store::Store,
- ClaimGeneratorInfo, Ingredient, ManifestAssertion, ManifestAssertionKind, RemoteSigner, Signer,
+ AsyncSigner, ClaimGeneratorInfo, Ingredient, ManifestAssertion, ManifestAssertionKind,
+ RemoteSigner, Signer,
};
/// A Manifest represents all the information in a c2pa manifest
@@ -1148,11 +1150,81 @@ impl Manifest {
.await
}
+ /// Removes any existing manifest from a file
+ ///
+ /// This should only be used for special cases, such as converting an embedded manifest
+ /// to a cloud manifest
#[cfg(feature = "file_io")]
pub fn remove_manifest<P: AsRef<Path>>(asset_path: P) -> Result<()> {
use crate::jumbf_io::remove_jumbf_from_file;
remove_jumbf_from_file(asset_path.as_ref())
}
+
+ /// Generates a data hashed placeholder manifest for a file
+ ///
+ /// The return value is pre-formatted for insertion into a file of the given format
+ /// For JPEG it is a series of App11 JPEG segments containing space for a manifest
+ /// This is used to create a properly formatted file ready for signing
+ pub fn data_hash_placeholder(
+ &mut self,
+ signer: &dyn AsyncSigner,
+ format: &str,
+ ) -> Result<Vec<u8>> {
+ let dh: Result<DataHash> = self.find_assertion(DataHash::LABEL);
+ if dh.is_err() {
+ let mut ph = DataHash::new("jumbf manifest", "sha256");
+ for _ in 0..10 {
+ ph.add_exclusion(HashRange::new(0, 2));
+ }
+ let data = vec![1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10];
+ let mut stream = std::io::Cursor::new(data);
+ ph.gen_hash_from_stream(&mut stream)?;
+ self.add_assertion(&ph)?;
+ }
+ let mut store = self.to_store()?;
+ let placeholder = store.get_data_hashed_manifest_placeholder(signer, format)?;
+ Ok(placeholder)
+ }
+
+ /// Generates an data hashed embeddable manifest for a file
+ ///
+ /// The return value is pre-formatted for insertion into a file of the given format
+ /// For JPEG it is a series of App11 JPEG segments containing a signed manifest
+ /// This can directly replace a placeholder manifest to create a properly signed asset
+ /// The data hash must contain exclusions and may contain pre-calculated hashes
+ /// if an asset reader is provided, it will be used to calculate the data hash
+ pub async fn data_hash_embeddable_manifest(
+ &mut self,
+ dh: &DataHash,
+ signer: &dyn AsyncSigner,
+ format: &str,
+ mut asset_reader: Option<&mut dyn CAIRead>,
+ ) -> Result<Vec<u8>> {
+ let mut store = self.to_store()?;
+ if let Some(asset_reader) = asset_reader.as_deref_mut() {
+ asset_reader.rewind()?;
+ }
+ let cm = store
+ .get_data_hashed_embeddable_manifest(dh, signer, format, asset_reader)
+ .await?;
+ Ok(cm)
+ }
+
+ /// Generates a signed box hashed manifest, optionally preformatted for embedding
+ ///
+ /// The manifest must include a box hash assertion with correct hashes
+ pub async fn box_hash_embeddable_manifest(
+ &mut self,
+ signer: &dyn AsyncSigner,
+ format: Option<&str>,
+ ) -> Result<Vec<u8>> {
+ let mut store = self.to_store()?;
+ let mut cm = store.get_box_hashed_embeddable_manifest(signer).await?;
+ if let Some(format) = format {
+ cm = store.get_composed_manifest(&cm, format)?;
+ }
+ Ok(cm)
+ }
}
impl std::fmt::Display for Manifest {
@@ -1191,20 +1263,24 @@ pub(crate) mod tests {
#[cfg(target_arch = "wasm32")]
wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser);
+ use crate::{
+ assertions::{c2pa_action, Action, Actions},
+ utils::test::{temp_signer, TEST_VC},
+ Ingredient, Manifest, Result,
+ };
#[cfg(feature = "file_io")]
use crate::{
- assertions::labels::ACTIONS,
+ assertions::{labels::ACTIONS, DataHash},
error::Error,
+ hash_utils::HashRange,
resource_store::ResourceRef,
status_tracker::{DetailedStatusTracker, StatusTracker},
store::Store,
- utils::test::{fixture_path, temp_dir_path, temp_fixture_path, TEST_SMALL_JPEG},
- validation_status,
- };
- use crate::{
- assertions::{c2pa_action, Action, Actions},
- utils::test::{temp_signer, TEST_VC},
- Ingredient, Manifest, Result,
+ utils::test::{
+ fixture_path, temp_dir_path, temp_fixture_path, write_jpeg_placeholder_file,
+ TEST_SMALL_JPEG,
+ },
+ validation_status, SigningAlg,
};
// example of random data structure as an assertion
@@ -2103,4 +2179,88 @@ pub(crate) mod tests {
.embed(&output, &output, signer.as_ref())
.expect("embed");
}
+
+ #[actix::test]
+ #[cfg(feature = "file_io")]
+ async fn test_data_hash_embeddable_manifest() {
+ let ap = fixture_path("cloud.jpg");
+
+ let signer = crate::openssl::temp_signer_async::AsyncSignerAdapter::new(SigningAlg::Ps256);
+
+ let mut manifest = Manifest::new("claim_generator");
+
+ // get a placeholder the manifest
+ let placeholder = manifest.data_hash_placeholder(&signer, "jpeg").unwrap();
+
+ let temp_dir = tempfile::tempdir().unwrap();
+ let output = temp_dir_path(&temp_dir, "boxhash-out.jpg");
+ let mut output_file = std::fs::OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .open(&output)
+ .unwrap();
+
+ // write a jpeg file with a placeholder for the manifest (returns offset of the placeholder)
+ let offset =
+ write_jpeg_placeholder_file(&placeholder, &ap, &mut output_file, None).unwrap();
+
+ // build manifest to insert in the hole
+
+ // create an hash exclusion for the manifest
+ let exclusion = HashRange::new(offset, placeholder.len());
+ let exclusions = vec![exclusion];
+
+ let mut dh = DataHash::new("source_hash", "sha256");
+ dh.exclusions = Some(exclusions);
+
+ let signed_manifest = manifest
+ .data_hash_embeddable_manifest(&dh, &signer, "image/jpeg", Some(&mut output_file))
+ .await
+ .unwrap();
+
+ use std::io::{Seek, SeekFrom, Write};
+
+ // path in new composed manifest
+ output_file.seek(SeekFrom::Start(offset as u64)).unwrap();
+ output_file.write_all(&signed_manifest).unwrap();
+
+ let manifest_store = crate::ManifestStore::from_file(&output).expect("from_file");
+ println!("{manifest_store}");
+ assert!(manifest_store.validation_status().is_none());
+ }
+
+ #[actix::test]
+ #[cfg(feature = "file_io")]
+ async fn test_box_hash_embedable_manifest() {
+ let asset_bytes = include_bytes!("../tests/fixtures/CA.jpg");
+ let box_hash_data = include_bytes!("../tests/fixtures/boxhash.json");
+ let box_hash: crate::assertions::BoxHash = serde_json::from_slice(box_hash_data).unwrap();
+
+ let mut manifest = Manifest::new("test_app".to_owned());
+ manifest.set_title("BoxHashTest").set_format("image/jpeg");
+
+ manifest
+ .add_labeled_assertion(crate::assertions::labels::BOX_HASH, &box_hash)
+ .unwrap();
+
+ let signer = crate::openssl::temp_signer_async::AsyncSignerAdapter::new(SigningAlg::Ps256);
+
+ let embeddable = manifest
+ .box_hash_embeddable_manifest(&signer, None)
+ .await
+ .expect("embeddable_manifest");
+
+ // Validate the embeddable manifest against the asset bytes
+ let manifest_store = crate::ManifestStore::from_manifest_and_asset_bytes_async(
+ &embeddable,
+ "image/jpeg",
+ asset_bytes,
+ )
+ .await
+ .unwrap();
+ assert!(!manifest_store.manifests().is_empty());
+ assert!(manifest_store.validation_status().is_none());
+ println!("{manifest_store}");
+ }
}
diff --git a/sdk/src/openssl/temp_signer.rs b/sdk/src/openssl/temp_signer.rs
@@ -68,7 +68,7 @@ pub fn get_ec_signer<P: AsRef<Path>>(
match alg {
SigningAlg::Es256 | SigningAlg::Es384 | SigningAlg::Es512 => (),
_ => {
- panic!("Unknown EC signer alg {:#?}", alg);
+ panic!("Unknown EC signer alg {alg:#?}");
}
}
@@ -111,7 +111,7 @@ pub fn get_ed_signer<P: AsRef<Path>>(
tsa_url: Option<String>,
) -> (EdSigner, PathBuf) {
if alg != SigningAlg::Ed25519 {
- panic!("Unknown ED signer alg {:#?}", alg);
+ panic!("Unknown ED signer alg {alg:#?}");
}
let mut sign_cert_path = path.as_ref().to_path_buf();
@@ -155,7 +155,7 @@ pub fn get_rsa_signer<P: AsRef<Path>>(
match alg {
SigningAlg::Ps256 | SigningAlg::Ps384 | SigningAlg::Ps512 => (),
_ => {
- panic!("Unknown RSA signer alg {:#?}", alg);
+ panic!("Unknown RSA signer alg {alg:#?}");
}
}
diff --git a/sdk/src/store.rs b/sdk/src/store.rs
@@ -45,6 +45,7 @@ use crate::{
get_assetio_handler, load_jumbf_from_stream, object_locations_from_stream,
save_jumbf_to_memory, save_jumbf_to_stream,
},
+ salt::DefaultSalt,
status_tracker::{log_item, OneShotStatusTracker, StatusTracker},
utils::{
hash_utils::{hash256, HashRange},
@@ -1619,14 +1620,59 @@ impl Store {
Ok(())
}
- /// Returns a manifest suitible for direct embedding by a client. The
- /// manfiest are only supported for cases when the client has provided
- /// a content hash binding. Note, will not work for cases like BMFF where
- /// the position of the content is also encoded.
- pub fn get_embeddable_manifest(&mut self, signer: &dyn Signer) -> Result<Vec<u8>> {
- let mut jumbf_bytes = self.to_jumbf_internal(signer.reserve_size())?;
+ /// This function is used to pre-generate a manifest with place holders for the final
+ /// DataHash and Manifest Signature. The DataHash will reserve space for at least 10
+ /// Exclusion ranges. The Signature box reserved size is based on the size returned by
+ /// the Signer. This function is not needed when using Box Hash. This function is used
+ /// in conjunction with `get_data_hashed_embeddable_manifest`. The manifest returned
+ /// from `get_data_hashed_embeddable_manifest` will have a size that matches this function.
+ pub fn get_data_hashed_manifest_placeholder(
+ &mut self,
+ signer: &dyn AsyncSigner,
+ format: &str,
+ ) -> Result<Vec<u8>> {
+ let pc = self.provenance_claim_mut().ok_or(Error::ClaimEncoding)?;
- let pc = self.provenance_claim().ok_or(Error::ClaimEncoding)?;
+ // if user did not supply a hash
+ if pc.hash_assertions().is_empty() {
+ // create placholder DataHash large enough for 10 Exclusions
+ let mut ph = DataHash::new("jumbf manifest", pc.alg());
+ for _ in 0..10 {
+ ph.add_exclusion(HashRange::new(0, 2));
+ }
+ let data = vec![1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10];
+ let mut stream = Cursor::new(data);
+ ph.gen_hash_from_stream(&mut stream)?;
+
+ pc.add_assertion_with_salt(&ph, &DefaultSalt::default())?;
+ }
+
+ let jumbf_bytes = self.to_jumbf_internal(signer.reserve_size())?;
+
+ let composed = self.get_composed_manifest(&jumbf_bytes, format)?;
+
+ Ok(composed)
+ }
+
+ /// Returns a finalized, signed manifest. The manfiest are only supported
+ /// for cases when the client has provided a data hash content hash binding. Note,
+ /// this function will not work for cases like BMFF where the position
+ /// of the content is also encoded. This function is not compatible with
+ /// BMFF hash binding. If a BMFF data hash or box hash is detected that is
+ /// an error. The DataHash placeholder assertion will be adjusted to the contain
+ /// the correct values. If the asset_reader value is supplied it will also perform
+ /// the hash calulations, otherwise the function uses the caller supplied values.
+ /// It is an error if `get_data_hashed_manifest_placeholder` was not called first
+ /// as this call inserts the DataHash placeholder assertion to reserve space for the
+ /// actual hash values not required when using BoxHashes.
+ pub async fn get_data_hashed_embeddable_manifest(
+ &mut self,
+ dh: &DataHash,
+ signer: &dyn AsyncSigner,
+ format: &str,
+ asset_reader: Option<&mut dyn CAIRead>,
+ ) -> Result<Vec<u8>> {
+ let pc = self.provenance_claim_mut().ok_or(Error::ClaimEncoding)?;
// make sure there are data hashes present before generating
if pc.hash_assertions().is_empty() {
@@ -1642,8 +1688,64 @@ impl Store {
));
}
+ let mut adusted_dh = DataHash::new("jumbf manifest", pc.alg());
+ adusted_dh.exclusions = dh.exclusions.clone();
+ adusted_dh.hash = dh.hash.clone();
+
+ if let Some(reader) = asset_reader {
+ // calc hashes
+ adusted_dh.gen_hash_from_stream(reader)?;
+ }
+
+ // update the placeholder hash
+ pc.update_data_hash(adusted_dh)?;
+
+ // reborrow immuttable
+ let pc = self.provenance_claim().ok_or(Error::ClaimEncoding)?;
+ let mut jumbf_bytes = self.to_jumbf_internal(signer.reserve_size())?;
+
// sign contents
- let sig = self.sign_claim(pc, signer, signer.reserve_size())?;
+ let sig = self
+ .sign_claim_async(pc, signer, signer.reserve_size())
+ .await?;
+ let sig_placeholder = Store::sign_claim_placeholder(pc, signer.reserve_size());
+
+ if sig_placeholder.len() != sig.len() {
+ return Err(Error::CoseSigboxTooSmall);
+ }
+
+ patch_bytes(&mut jumbf_bytes, &sig_placeholder, &sig)
+ .map_err(|_| Error::JumbfCreationError)?;
+
+ self.get_composed_manifest(&jumbf_bytes, format)
+ }
+
+ /// Returns a finalized, signed manifest. The client is required to have
+ /// included the necessary box hash assertion with the pregenerated hashes.
+ pub async fn get_box_hashed_embeddable_manifest(
+ &mut self,
+ signer: &dyn AsyncSigner,
+ ) -> Result<Vec<u8>> {
+ let pc = self.provenance_claim().ok_or(Error::ClaimEncoding)?;
+
+ // make sure there is only one
+ if pc.hash_assertions().len() != 1 {
+ return Err(Error::BadParam(
+ "Claim must have exactly one hash binding assertion".to_string(),
+ ));
+ }
+
+ // only allow box hash assertions to be present
+ if pc.box_hash_assertions().is_empty() {
+ return Err(Error::BadParam("Missing box hash assertion".to_string()));
+ }
+
+ let mut jumbf_bytes = self.to_jumbf_internal(signer.reserve_size())?;
+
+ // sign contents
+ let sig = self
+ .sign_claim_async(pc, signer, signer.reserve_size())
+ .await?;
let sig_placeholder = Store::sign_claim_placeholder(pc, signer.reserve_size());
if sig_placeholder.len() != sig.len() {
@@ -1656,6 +1758,18 @@ impl Store {
Ok(jumbf_bytes)
}
+ /// Returns the supplied manifest composed to be directly compatibile with the desired format.
+ /// For example, if format is JPEG funtion will return the set of APP11 segments that contains
+ /// the manifest. Similarly for PNG it would be the PNG chunk complete with header and CRC.
+ pub fn get_composed_manifest(&self, manifest_bytes: &[u8], format: &str) -> Result<Vec<u8>> {
+ if let Some(h) = get_assetio_handler(format) {
+ if let Some(composed_data_handler) = h.composed_data_ref() {
+ return composed_data_handler.compose_manifest(manifest_bytes, format);
+ }
+ }
+ Err(Error::UnsupportedType)
+ }
+
/// Embed the claims store as jumbf into a stream. Updates XMP with provenance record.
/// When called, the stream should contain an asset matching format.
/// on return, the stream will contain the new manifest signed with signer
@@ -1960,7 +2074,8 @@ impl Store {
}
// we will not do automatic hashing if we detect a box hash present
- if pc.box_hash_assertions().is_empty() {
+ let mut needs_hashing = false;
+ if pc.hash_assertions().is_empty() {
// 2) Get hash ranges if needed, do not generate for update manifests
let mut hash_ranges = object_locations_from_stream(format, &mut intermediate_stream)?;
let hashes: Vec<DataHash> = if pc.update_manifest() {
@@ -1970,7 +2085,7 @@ impl Store {
&mut intermediate_stream,
pc.alg(),
&mut hash_ranges,
- false,
+ true,
)?
};
@@ -1982,6 +2097,7 @@ impl Store {
pc.add_assertion(&hash)?;
}
+ needs_hashing = true;
}
// 3) Generate in memory CAI jumbf block
@@ -1995,10 +2111,9 @@ impl Store {
// 4) determine final object locations and patch the asset hashes with correct offset
// replace the source with correct asset hashes so that the claim hash will be correct
- let pc = self.provenance_claim_mut().ok_or(Error::ClaimEncoding)?;
+ if needs_hashing {
+ let pc = self.provenance_claim_mut().ok_or(Error::ClaimEncoding)?;
- // we will not do automatic hashing if we detect a box hash present
- if pc.box_hash_assertions().is_empty() {
// get the final hash ranges, but not for update manifests
intermediate_stream.rewind()?;
output_stream.rewind()?;
@@ -2185,6 +2300,7 @@ impl Store {
}
} else {
// we will not do automatic hashing if we detect a box hash present
+ let mut needs_hashing = false;
if pc.box_hash_assertions().is_empty() {
// 2) Get hash ranges if needed, do not generate for update manifests
let mut hash_ranges = object_locations(&output_path)?;
@@ -2202,6 +2318,7 @@ impl Store {
pc.add_assertion(&hash)?;
}
+ needs_hashing = true;
}
// 3) Generate in memory CAI jumbf block
@@ -2214,8 +2331,9 @@ impl Store {
// 4) determine final object locations and patch the asset hashes with correct offset
// replace the source with correct asset hashes so that the claim hash will be correct
// If box hash is present we don't do any other
- let pc = self.provenance_claim_mut().ok_or(Error::ClaimEncoding)?;
- if pc.box_hash_assertions().is_empty() {
+ if needs_hashing {
+ let pc = self.provenance_claim_mut().ok_or(Error::ClaimEncoding)?;
+
// get the final hash ranges, but not for update manifests
let mut new_hash_ranges = object_locations(&output_path)?;
let updated_hashes = if pc.update_manifest() {
@@ -2759,6 +2877,9 @@ pub mod tests {
#![allow(clippy::panic)]
#![allow(clippy::unwrap_used)]
+ use std::io::Write;
+
+ use sha2::{Digest, Sha256};
use tempfile::tempdir;
use twoway::find_bytes;
@@ -2772,9 +2893,11 @@ pub mod tests {
},
status_tracker::*,
utils::{
+ hash_utils::Hasher,
patch::patch_file,
test::{
create_test_claim, fixture_path, temp_dir_path, temp_fixture_path, temp_signer,
+ write_jpeg_placeholder_file,
},
},
AssertionJson, SigningAlg,
@@ -3998,25 +4121,25 @@ pub mod tests {
}
/*
- #[test]
- fn test_bmff_fragments() {
- let init_stream_path = fixture_path("dashinit.mp4");
- let segment_stream_path = fixture_path("dash1.m4s");
-
- let init_stream = std::fs::read(init_stream_path).unwrap();
- let segment_stream = std::fs::read(segment_stream_path).unwrap();
-
- let mut report = DetailedStatusTracker::new();
- let store = Store::load_fragment_from_memory(
- "mp4",
- &init_stream,
- &segment_stream,
- true,
- &mut report,
- )
- .expect("load_from_asset");
- println!("store = {store}");
- }
+ #[test]
+ fn test_bmff_fragments() {
+ let init_stream_path = fixture_path("dashinit.mp4");
+ let segment_stream_path = fixture_path("dash1.m4s");
+
+ let init_stream = std::fs::read(init_stream_path).unwrap();
+ let segment_stream = std::fs::read(segment_stream_path).unwrap();
+
+ let mut report = DetailedStatusTracker::new();
+ let store = Store::load_fragment_from_memory(
+ "mp4",
+ &init_stream,
+ &segment_stream,
+ true,
+ &mut report,
+ )
+ .expect("load_from_asset");
+ println!("store = {store}");
+ }
*/
#[test]
@@ -4256,7 +4379,7 @@ pub mod tests {
Ok(_store) => panic!("did not expect to have a store"),
Err(e) => match e {
Error::JumbfNotFound => {}
- e => panic!("unexpected error: {}", e),
+ e => panic!("unexpected error: {e}"),
},
}
}
@@ -4337,6 +4460,9 @@ pub mod tests {
// read from new file
let new_store = Store::load_from_asset(&op, true, &mut report).unwrap();
+ let errors = report_split_errors(report.get_log_mut());
+ assert!(errors.is_empty());
+
// dump store and compare to original
for claim in new_store.claims() {
let _restored_json = claim
@@ -4365,10 +4491,10 @@ pub mod tests {
}
}
- #[test]
- fn test_embeddable_manifest() {
+ #[actix::test]
+ async fn test_boxhash_embeddable_manifest() {
// test adding to actual image
- let ap = fixture_path("CA.jpg");
+ let ap = fixture_path("boxhash.jpg");
let box_hash_path = fixture_path("boxhash.json");
// Create claims store.
@@ -4386,23 +4512,187 @@ pub mod tests {
store.commit_claim(claim).unwrap();
// Do we generate JUMBF?
- let signer = temp_signer();
+ let signer = crate::openssl::temp_signer_async::AsyncSignerAdapter::new(SigningAlg::Ps256);
// get the embeddable manifest
- let em = store.get_embeddable_manifest(signer.as_ref()).unwrap();
+ let em = store
+ .get_box_hashed_embeddable_manifest(&signer)
+ .await
+ .unwrap();
+
+ // get composed version for embedding to JPEG
+ let cm = store.get_composed_manifest(&em, "jpg").unwrap();
+
+ // insert manifest into ouput asset
+ let jpeg_io = get_assetio_handler_from_path(&ap).unwrap();
+ let ol = jpeg_io.get_object_locations(&ap).unwrap();
+
+ let cai_loc = ol
+ .iter()
+ .find(|o| o.htype == HashBlockObjectType::Cai)
+ .unwrap();
+
+ // remove any existing manifest
+ jpeg_io.read_cai_store(&ap).unwrap();
+
+ // build new asset in memory inserting new manifest
+ let outbuf = Vec::new();
+ let mut out_stream = Cursor::new(outbuf);
+ let mut input_file = std::fs::File::open(&ap).unwrap();
+
+ // write before
+ let mut before = vec![0u8; cai_loc.offset];
+ input_file.read_exact(before.as_mut_slice()).unwrap();
+ out_stream.write_all(&before).unwrap();
+
+ // write composed bytes
+ out_stream.write_all(&cm).unwrap();
+
+ // write bytes after
+ let mut after_buf = Vec::new();
+ input_file.read_to_end(&mut after_buf).unwrap();
+ out_stream.write_all(&after_buf).unwrap();
+
+ // save to output file
+ let temp_dir = tempfile::tempdir().unwrap();
+ let output = temp_dir_path(&temp_dir, "boxhash-out.jpg");
+ let mut output_file = std::fs::OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .open(&output)
+ .unwrap();
+ output_file.write_all(&out_stream.into_inner()).unwrap();
let mut report = DetailedStatusTracker::new();
- let new_store = Store::from_jumbf(&em, &mut report).unwrap();
+ let _new_store = Store::load_from_asset(&output, true, &mut report).unwrap();
- let pc = new_store.provenance_claim().unwrap();
- let bhp = get_assetio_handler_from_path(&ap)
- .unwrap()
- .asset_box_hash_ref()
+ let errors = report_split_errors(report.get_log_mut());
+ assert!(errors.is_empty());
+ }
+
+ #[actix::test]
+ async fn test_datahash_embeddable_manifest() {
+ // test adding to actual image
+ let ap = fixture_path("cloud.jpg");
+
+ // Do we generate JUMBF?
+ let signer = crate::openssl::temp_signer_async::AsyncSignerAdapter::new(SigningAlg::Ps256);
+
+ // Create claims store.
+ let mut store = Store::new();
+
+ // Create a new claim.
+ let claim = create_test_claim().unwrap();
+
+ store.commit_claim(claim).unwrap();
+
+ // get a placeholder the manifest
+ let placeholder = store
+ .get_data_hashed_manifest_placeholder(&signer, "jpeg")
.unwrap();
- for h in pc.box_hash_assertions() {
- let bh = BoxHash::from_assertion(h).unwrap();
- bh.verify_hash(&ap, None, bhp).unwrap();
- }
+ let temp_dir = tempfile::tempdir().unwrap();
+ let output = temp_dir_path(&temp_dir, "boxhash-out.jpg");
+ let mut output_file = std::fs::OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .open(&output)
+ .unwrap();
+
+ // write a jpeg file with a placeholder for the manifest (returns offset of the placeholder)
+ let offset =
+ write_jpeg_placeholder_file(&placeholder, &ap, &mut output_file, None).unwrap();
+
+ // build manifest to insert in the hole
+
+ // create an hash exclusion for the manifest
+ let exclusion = HashRange::new(offset, placeholder.len());
+ let exclusions = vec![exclusion];
+
+ let mut dh = DataHash::new("source_hash", "sha256");
+ dh.exclusions = Some(exclusions);
+
+ // get the embeddable manifest, letting API do the hashing
+ output_file.rewind().unwrap();
+ let cm = store
+ .get_data_hashed_embeddable_manifest(&dh, &signer, "jpeg", Some(&mut output_file))
+ .await
+ .unwrap();
+
+ // path in new composed manifest
+ output_file.seek(SeekFrom::Start(offset as u64)).unwrap();
+ output_file.write_all(&cm).unwrap();
+
+ let mut report = DetailedStatusTracker::new();
+ let _new_store = Store::load_from_asset(&output, true, &mut report).unwrap();
+
+ let errors = report_split_errors(report.get_log_mut());
+ assert!(errors.is_empty());
+ }
+
+ #[actix::test]
+ async fn test_datahash_embeddable_manifest_user_hashed() {
+ // test adding to actual image
+ let ap = fixture_path("cloud.jpg");
+
+ let mut hasher = Hasher::SHA256(Sha256::new());
+
+ // Do we generate JUMBF?
+ let signer = crate::openssl::temp_signer_async::AsyncSignerAdapter::new(SigningAlg::Ps256);
+
+ // Create claims store.
+ let mut store = Store::new();
+
+ // Create a new claim.
+ let claim = create_test_claim().unwrap();
+
+ store.commit_claim(claim).unwrap();
+
+ // get a placeholder for the manifest
+ let placeholder = store
+ .get_data_hashed_manifest_placeholder(&signer, "jpeg")
+ .unwrap();
+
+ let temp_dir = tempfile::tempdir().unwrap();
+ let output = temp_dir_path(&temp_dir, "boxhash-out.jpg");
+ let mut output_file = std::fs::OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .open(&output)
+ .unwrap();
+
+ // write a jpeg file with a placeholder for the manifest (returns offset of the placeholder)
+ let offset =
+ write_jpeg_placeholder_file(&placeholder, &ap, &mut output_file, Some(&mut hasher))
+ .unwrap();
+
+ // create target data hash
+ // create an hash exclusion for the manifest
+ let exclusion = HashRange::new(offset, placeholder.len());
+ let exclusions = vec![exclusion];
+
+ //input_file.rewind().unwrap();
+ let mut dh = DataHash::new("source_hash", "sha256");
+ dh.hash = Hasher::finalize(hasher);
+ dh.exclusions = Some(exclusions);
+
+ // get the embeddable manifest, using user hashing
+ let cm = store
+ .get_data_hashed_embeddable_manifest(&dh, &signer, "jpeg", None)
+ .await
+ .unwrap();
+
+ // path in new composed manifest
+ output_file.seek(SeekFrom::Start(offset as u64)).unwrap();
+ output_file.write_all(&cm).unwrap();
+
+ let mut report = DetailedStatusTracker::new();
+ let _new_store = Store::load_from_asset(&output, true, &mut report).unwrap();
+
+ let errors = report_split_errors(report.get_log_mut());
+ assert!(errors.is_empty());
}
}
diff --git a/sdk/src/utils/hash_utils.rs b/sdk/src/utils/hash_utils.rs
@@ -272,7 +272,8 @@ where
// merge standard ranges and BMFF V2 ranges into single list
if !bmff_v2_starts.is_empty() {
// remove any offset hashes that would be excluded
- bmff_v2_starts.retain(|o| ranges.iter().any(|r| *o + 1 == r));
+ let test_ranges = ranges.clone().into_smallvec();
+ bmff_v2_starts.retain(|o| test_ranges.iter().any(|r| r.contains(&(*o + 1))));
// add in remaining BMFF V2 offsets
for os in bmff_v2_starts.iter() {
diff --git a/sdk/src/utils/test.rs b/sdk/src/utils/test.rs
@@ -7,18 +7,21 @@
// Unless required by applicable law or agreed to in writing,
// this software is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR REPRESENTATIONS OF ANY KIND, either express or
-// implied. See the LICENSE-MIT and LICENSE-APACHE files for the
+// implied. See the LICENSE-MIT and LICENSE-APACHE files for thema
// specific language governing permissions and limitations under
// each license.
#![allow(clippy::unwrap_used)]
use std::path::PathBuf;
+#[cfg(feature = "file_io")]
+use std::{
+ io::{Cursor, Read, Write},
+ path::Path,
+};
use tempfile::TempDir;
-#[cfg(feature = "file_io")]
-use crate::create_signer;
use crate::{
assertions::{labels, Action, Actions, Ingredient, ReviewRating, SchemaDotOrg, Thumbnail},
claim::Claim,
@@ -26,6 +29,11 @@ use crate::{
store::Store,
Result, Signer, SigningAlg,
};
+#[cfg(feature = "file_io")]
+use crate::{
+ asset_io::CAIReadWrite, create_signer, hash_utils::Hasher,
+ jumbf_io::get_assetio_handler_from_path,
+};
#[cfg(feature = "openssl_sign")]
use crate::{openssl::RsaSigner, signer::ConfigurableSigner};
@@ -212,6 +220,51 @@ pub fn temp_signer_file() -> RsaSigner {
.expect("get_temp_signer")
}
+/// Utility to create a test file with a placeholder for a manifest
+#[cfg(feature = "file_io")]
+pub fn write_jpeg_placeholder_file(
+ placeholder: &[u8],
+ input: &Path,
+ output_file: &mut dyn CAIReadWrite,
+ mut hasher: Option<&mut Hasher>,
+) -> Result<usize> {
+ // get where we will put the data
+ let mut f = std::fs::File::open(input).unwrap();
+ let jpeg_io = get_assetio_handler_from_path(input).unwrap();
+ let box_mapper = jpeg_io.asset_box_hash_ref().unwrap();
+ let boxes = box_mapper.get_box_map(&mut f).unwrap();
+ let sof = boxes.iter().find(|b| b.names[0] == "SOF0").unwrap();
+
+ // build new asset with hole for new manifest
+ let outbuf = Vec::new();
+ let mut out_stream = Cursor::new(outbuf);
+ let mut input_file = std::fs::File::open(input).unwrap();
+
+ // write before
+ let mut before = vec![0u8; sof.range_start];
+ input_file.read_exact(before.as_mut_slice()).unwrap();
+ if let Some(hasher) = hasher.as_deref_mut() {
+ hasher.update(&before);
+ }
+ out_stream.write_all(&before).unwrap();
+
+ // write placeholder
+ out_stream.write_all(placeholder).unwrap();
+
+ // write bytes after
+ let mut after_buf = Vec::new();
+ input_file.read_to_end(&mut after_buf).unwrap();
+ if let Some(hasher) = hasher {
+ hasher.update(&after_buf);
+ }
+ out_stream.write_all(&after_buf).unwrap();
+
+ // save to output file
+ output_file.write_all(&out_stream.into_inner()).unwrap();
+
+ Ok(sof.range_start)
+}
+
pub(crate) struct TestGoodSigner {}
impl crate::Signer for TestGoodSigner {
fn sign(&self, _data: &[u8]) -> Result<Vec<u8>> {
diff --git a/sdk/tests/fixtures/boxhash.jpg b/sdk/tests/fixtures/boxhash.jpg
Binary files differ.
