commit 8f555302c2be26f258b3b9019cdd01f88056e447
parent c3eb198d4e11f9f6cf2e2d7fc6574ef5598cf091
Author: mauricefisher64 <92736594+mauricefisher64@users.noreply.github.com>
Date: Thu, 18 May 2023 09:40:19 -0400
Riff streaming support (#248)
* Initial RIFF implementation
* AVI test
* webp support
* Disable XMP default writting
* Fix broken hashing
* Refactor asset handlers to control supported features
* Fix clippy errors
* Moved XMP generation to the asset handlers
* Remove XMP stuff from store since it is now performed by handlders
* XMP restrict some file types
BMFF generation bug for remote manifests
* Code cleanup
* Update version
* format issues
* more formatting fixes
* cleanup
* Bump major number since new error codes are available
* Fix PR comments
* Minor cleanup
* Support HEIC, HEIF, & AVIF
* PR comment fixes
* Add comment for future enhancement
* Streaming writes
* Clippy cleanup
* Add XMP generation for WEBP
* Fix for typo
* Unit test fixes/changes
* cargo formatting
* More formatting
* format again
* PR fixes
* Changes based on PR comments
Diffstat:
11 files changed, 566 insertions(+), 108 deletions(-)
diff --git a/sdk/src/asset_handlers/jpeg_io.rs b/sdk/src/asset_handlers/jpeg_io.rs
@@ -93,11 +93,10 @@ fn add_required_segs_to_stream(
let aio = JpegIO {};
aio.write_cai(input_stream, output_stream, &no_bytes)?;
} else {
- // just move input to output
- let mut buf: Vec<u8> = Vec::new();
+ // just clone
input_stream.rewind()?;
- input_stream.read_to_end(&mut buf).map_err(Error::IoError)?;
- output_stream.write_all(&buf)?;
+ output_stream.rewind()?;
+ std::io::copy(input_stream, output_stream)?;
}
} else {
return Err(Error::UnsupportedType);
diff --git a/sdk/src/asset_handlers/png_io.rs b/sdk/src/asset_handlers/png_io.rs
@@ -159,11 +159,10 @@ fn add_required_chunks_to_stream(
let aio = PngIO {};
aio.write_cai(input_stream, output_stream, &no_bytes)?;
} else {
- // just move input to output
- let mut buf: Vec<u8> = Vec::new();
+ // just clone
input_stream.rewind()?;
- input_stream.read_to_end(&mut buf).map_err(Error::IoError)?;
- output_stream.write_all(&buf)?;
+ output_stream.rewind()?;
+ std::io::copy(input_stream, output_stream)?;
}
} else {
return Err(Error::UnsupportedType);
diff --git a/sdk/src/asset_handlers/riff_io.rs b/sdk/src/asset_handlers/riff_io.rs
@@ -17,16 +17,17 @@ use std::{
path::Path,
};
+use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use conv::ValueFrom;
use riff::*;
use crate::{
asset_io::{
- AssetIO, AssetPatch, CAIRead, CAIReadWrite, CAIReader, HashBlockObjectType,
- HashObjectPositions, RemoteRefEmbed, RemoteRefEmbedType,
+ AssetIO, AssetPatch, CAIRead, CAIReadWrapper, CAIReadWrite, CAIReadWriteWrapper, CAIReader,
+ CAIWriter, HashBlockObjectType, HashObjectPositions, RemoteRefEmbed, RemoteRefEmbedType,
},
error::{Error, Result},
- jumbf_io::get_file_extension,
+ utils::xmp_inmemory_utils::{add_provenance, MIN_XMP},
};
static SUPPORTED_TYPES: [&str; 9] = [
@@ -50,6 +51,24 @@ const C2PA_CHUNK_ID: ChunkId = ChunkId {
value: [0x43, 0x32, 0x50, 0x41],
}; // C2PA
+const VP8X_ID: ChunkId = ChunkId {
+ value: [0x56, 0x50, 0x38, 0x58],
+}; // VP8X chunk to hold auxiliary info
+
+const VP8_ID: ChunkId = ChunkId {
+ value: [0x56, 0x50, 0x38, 0x20],
+}; // VP8 chunk
+
+const VP8L_ID: ChunkId = ChunkId {
+ value: [0x56, 0x50, 0x38, 0x4c],
+}; // VP8L chunk
+
+const XMP_CHUNK_ID: ChunkId = ChunkId {
+ value: [0x58, 0x4d, 0x50, 0x20],
+}; // XMP
+
+const XMP_FLAG: u32 = 4;
+
fn read_items<T>(iter: &mut T) -> Vec<T::Item>
where
T: Iterator,
@@ -61,54 +80,159 @@ where
vec
}
-fn inject_c2pa<T>(chunk: &Chunk, file: &mut T, data: &[u8], format: &str) -> Result<ChunkContents>
+fn get_height_and_width(chunk_contents: &[ChunkContents]) -> Result<(u16, u16)> {
+ if let Some(ChunkContents::Data(_id, chunk_data)) = chunk_contents.iter().find(|c| match c {
+ ChunkContents::Data(id, _) => *id == VP8L_ID,
+ _ => false,
+ }) {
+ let mut chunk_stream = Cursor::new(chunk_data);
+ chunk_stream.seek(SeekFrom::Start(1))?; // skip signature byte
+
+ // width and length are 12 bits packed together
+ let first_bytes = chunk_stream.read_u16::<LittleEndian>()?;
+ let width = 1 + (first_bytes & 0x3fff); // add 1 for VP8L
+ let last_two = (first_bytes & 0xc000) >> 14; // last two bits of first bytes are first 2 of height
+ let height = 1 + (((chunk_stream.read_u16::<LittleEndian>()? & 0xfff) << 2) | last_two);
+
+ return Ok((height, width));
+ }
+
+ if let Some(ChunkContents::Data(_id, chunk_data)) = chunk_contents.iter().find(|c| match c {
+ ChunkContents::Data(id, _) => *id == VP8_ID,
+ _ => false,
+ }) {
+ let mut chunk_stream = Cursor::new(chunk_data);
+ chunk_stream.seek(SeekFrom::Start(6))?; // skip frame tag and start code
+
+ let width = chunk_stream.read_u16::<LittleEndian>()? & 0x3fff;
+ let height = chunk_stream.read_u16::<LittleEndian>()? & 0x3fff;
+
+ return Ok((height, width));
+ }
+
+ Err(Error::InvalidAsset(
+ "WEBP missing VP8 or VP8L segment".to_string(),
+ ))
+}
+
+fn inject_c2pa<T>(
+ chunk: &Chunk,
+ stream: &mut T,
+ data: &[u8],
+ xmp_data: Option<&[u8]>,
+ format: &str,
+) -> Result<ChunkContents>
where
T: std::io::Seek + std::io::Read,
{
let id = chunk.id();
let is_riff_chunk: bool = id == riff::RIFF_ID;
+ stream.rewind()?;
if is_riff_chunk || id == riff::LIST_ID {
- let chunk_type = chunk.read_type(file).map_err(|_| {
+ let chunk_type = chunk.read_type(stream).map_err(|_| {
Error::InvalidAsset("RIFF handler could not parse file format {format}".to_string())
})?;
- let mut children = read_items(&mut chunk.iter(file));
+ let mut children = read_items(&mut chunk.iter(stream));
let mut children_contents: Vec<ChunkContents> = Vec::new();
- if is_riff_chunk {
+ if is_riff_chunk && !data.is_empty() {
// remove c2pa manifest store in RIFF chunk
children.retain(|c| c.id() != C2PA_CHUNK_ID);
}
- // for non webp we can place at the front
- // add c2pa manifest
- if is_riff_chunk && !data.is_empty() && !format.contains("webp") {
- children_contents.push(ChunkContents::Data(C2PA_CHUNK_ID, data.to_vec()));
+ if is_riff_chunk && xmp_data.is_some() {
+ // remove XMP in RIFF chunk so we can replace
+ children.retain(|c| c.id() != XMP_CHUNK_ID);
}
+ // duplicate all top level children
for child in children {
- children_contents.push(inject_c2pa(&child, file, data, format)?);
+ children_contents.push(inject_c2pa(&child, stream, data, xmp_data, format)?);
+ }
+
+ // add XMP if needed
+ if let Some(xmp) = xmp_data {
+ if is_riff_chunk && !xmp.is_empty() {
+ // if this is a webp doc we must also update VP8X
+ if format == "webp" {
+ // if already present we can patch otherwise add
+ if let Some(ChunkContents::Data(_id, chunk_data)) =
+ children_contents.iter_mut().find(|c| match c {
+ ChunkContents::Data(id, _) => *id == VP8X_ID,
+ _ => false,
+ })
+ {
+ let mut chunk_stream = Cursor::new(chunk_data);
+
+ let mut flags = chunk_stream.read_u32::<LittleEndian>()?;
+
+ // add in XMP flag
+ flags |= XMP_FLAG;
+
+ chunk_stream.rewind()?;
+
+ // write back changes
+ chunk_stream.write_u32::<LittleEndian>(flags)?;
+ } else {
+ // add new VP8X
+
+ // get height and width from VBL
+ if let Ok((height, width)) = get_height_and_width(&children_contents) {
+ let data: Vec<u8> = Vec::new();
+ let mut chunk_writer = Cursor::new(data);
+
+ let flags: u32 = XMP_FLAG;
+ let vp8x_height = height as u32 - 1;
+ let vp8x_width = width as u32 - 1;
+
+ // write flags
+ chunk_writer.write_u32::<LittleEndian>(flags)?;
+
+ // write width then height
+ chunk_writer.write_u24::<LittleEndian>(vp8x_width)?;
+ chunk_writer.write_u24::<LittleEndian>(vp8x_height)?;
+
+ // make new VP8X chunk and prepend to children list
+ let mut tmp_vec: Vec<ChunkContents> = Vec::new();
+ tmp_vec.push(ChunkContents::Data(VP8X_ID, chunk_writer.into_inner()));
+ tmp_vec.extend(children_contents);
+ children_contents = tmp_vec;
+ } else {
+ return Err(Error::InvalidAsset(
+ "Could not parse VP8 or VP8L".to_string(),
+ ));
+ }
+ }
+ }
+
+ children_contents.push(ChunkContents::Data(XMP_CHUNK_ID, xmp.to_vec()));
+ }
}
- // for non webp we can place at the front
- // add c2pa manifest
- if is_riff_chunk && !data.is_empty() && format.contains("webp") {
- children_contents.push(ChunkContents::Data(C2PA_CHUNK_ID, data.to_vec()));
+ // place at the end for maximum compatibility
+ if is_riff_chunk && !data.is_empty() {
+ let mut d = data.to_vec();
+ if d.len() % 2 == 1 {
+ // must be even
+ d.push(0);
+ }
+ children_contents.push(ChunkContents::Data(C2PA_CHUNK_ID, d));
}
Ok(ChunkContents::Children(id, chunk_type, children_contents))
} else if id == riff::SEQT_ID {
- let children = read_items(&mut chunk.iter_no_type(file));
+ let children = read_items(&mut chunk.iter_no_type(stream));
let mut children_contents: Vec<ChunkContents> = Vec::new();
for child in children {
- children_contents.push(inject_c2pa(&child, file, data, format)?);
+ children_contents.push(inject_c2pa(&child, stream, data, xmp_data, format)?);
}
Ok(ChunkContents::ChildrenNoType(id, children_contents))
} else {
let contents = chunk
- .read_contents(file)
+ .read_contents(stream)
.map_err(|_| Error::InvalidAsset("RIFF handler could not parse file".to_string()))?;
Ok(ChunkContents::Data(id, contents))
}
@@ -134,12 +258,10 @@ fn get_manifest_pos(reader: &mut dyn CAIRead) -> Option<(u64, u32)> {
}
impl CAIReader for RiffIO {
- fn read_cai(&self, reader: &mut dyn CAIRead) -> Result<Vec<u8>> {
- let mut asset: Vec<u8> = Vec::new();
- reader.rewind()?;
- reader.read_to_end(&mut asset)?;
-
- let mut chunk_reader = Cursor::new(asset);
+ fn read_cai(&self, input_stream: &mut dyn CAIRead) -> Result<Vec<u8>> {
+ let mut chunk_reader = CAIReadWrapper {
+ reader: input_stream,
+ };
let top_level_chunks = riff::Chunk::read(&mut chunk_reader, 0)?;
@@ -149,7 +271,13 @@ impl CAIReader for RiffIO {
for c in top_level_chunks.iter(&mut chunk_reader) {
if c.id() == C2PA_CHUNK_ID {
- let output = c.read_contents(&mut chunk_reader)?;
+ let mut output = c.read_contents(&mut chunk_reader)?;
+ // the data may have been padded to account for even boundary requirement
+ if let Some(last_byte) = output.last() {
+ if *last_byte == 0 {
+ output.pop();
+ }
+ }
return Ok(output);
}
}
@@ -158,18 +286,54 @@ impl CAIReader for RiffIO {
}
// Get XMP block
- fn read_xmp(&self, _asset_reader: &mut dyn CAIRead) -> Option<String> {
- None // todo: figure out where XMP is stored for supported formats
+ fn read_xmp(&self, input_stream: &mut dyn CAIRead) -> Option<String> {
+ let top_level_chunks = {
+ let mut reader = CAIReadWrapper {
+ reader: input_stream,
+ };
+ Chunk::read(&mut reader, 0).ok()?
+ };
+
+ if top_level_chunks.id() != RIFF_ID {
+ return None;
+ }
+
+ let mut chunk_reader = CAIReadWrapper {
+ reader: input_stream,
+ };
+
+ for c in top_level_chunks.iter(&mut chunk_reader) {
+ if c.id() == XMP_CHUNK_ID {
+ let output = c.read_contents(&mut chunk_reader).ok()?;
+ let output_string = String::from_utf8_lossy(&output);
+
+ return Some(output_string.to_string());
+ }
+ }
+
+ None
}
}
-fn add_required_chunks(asset_path: &std::path::Path) -> Result<()> {
- let mut f = File::open(asset_path)?;
- let aio = RiffIO::new(&get_file_extension(asset_path).ok_or(Error::UnsupportedType)?);
-
- match aio.read_cai(&mut f) {
- Ok(_) => Ok(()),
- Err(_) => aio.save_cai_store(asset_path, &[1, 2, 3, 4]), // save arbitrary data
+fn add_required_chunks(
+ asset_type: &str,
+ input_stream: &mut dyn CAIRead,
+ output_stream: &mut dyn CAIReadWrite,
+) -> Result<()> {
+ let aio = RiffIO::new(asset_type);
+
+ match aio.read_cai(input_stream) {
+ Ok(_) => {
+ // just clone
+ input_stream.rewind()?;
+ output_stream.rewind()?;
+ std::io::copy(input_stream, output_stream)?;
+ Ok(())
+ }
+ Err(_) => {
+ input_stream.rewind()?;
+ aio.write_cai(input_stream, output_stream, &[1, 2, 3, 4]) // save arbitrary data
+ }
}
}
@@ -188,6 +352,10 @@ impl AssetIO for RiffIO {
self
}
+ fn get_writer(&self, asset_type: &str) -> Option<Box<dyn CAIWriter>> {
+ Some(Box::new(RiffIO::new(asset_type)))
+ }
+
fn asset_patch_ref(&self) -> Option<&dyn AssetPatch> {
Some(self)
}
@@ -198,46 +366,94 @@ impl AssetIO for RiffIO {
}
fn save_cai_store(&self, asset_path: &std::path::Path, store_bytes: &[u8]) -> Result<()> {
- let asset = std::fs::read(asset_path)?;
- let mut chunk_reader = Cursor::new(asset);
+ let mut input_stream = File::open(asset_path)?;
+
+ let mut output_stream = std::fs::OpenOptions::new()
+ .read(true)
+ .write(true)
+ .open(asset_path)
+ .map_err(Error::IoError)?;
+
+ self.write_cai(&mut input_stream, &mut output_stream, store_bytes)
+ }
+
+ fn get_object_locations(
+ &self,
+ asset_path: &std::path::Path,
+ ) -> Result<Vec<HashObjectPositions>> {
+ let mut f = std::fs::File::open(asset_path).map_err(|_err| Error::EmbeddingError)?;
+
+ self.get_object_locations_from_stream(&mut f)
+ }
+
+ fn remove_cai_store(&self, asset_path: &Path) -> Result<()> {
+ self.save_cai_store(asset_path, &[])
+ }
+
+ fn remote_ref_writer_ref(&self) -> Option<&dyn RemoteRefEmbed> {
+ Some(self)
+ }
- let top_level_chunks = Chunk::read(&mut chunk_reader, 0)?;
+ fn supported_types(&self) -> &[&str] {
+ &SUPPORTED_TYPES
+ }
+}
+
+impl CAIWriter for RiffIO {
+ fn write_cai(
+ &self,
+ input_stream: &mut dyn CAIRead,
+ output_stream: &mut dyn CAIReadWrite,
+ store_bytes: &[u8],
+ ) -> Result<()> {
+ let top_level_chunks = {
+ let mut reader = CAIReadWrapper {
+ reader: input_stream,
+ };
+ Chunk::read(&mut reader, 0)?
+ };
if top_level_chunks.id() != RIFF_ID {
return Err(Error::InvalidAsset("Invalid RIFF format".to_string()));
}
+ let mut reader = CAIReadWrapper {
+ reader: input_stream,
+ };
+
// replace/add manifest in memory
let new_contents = inject_c2pa(
&top_level_chunks,
- &mut chunk_reader,
+ &mut reader,
store_bytes,
+ None,
&self.riff_format,
)?;
+ let mut writer = CAIReadWriteWrapper {
+ reader_writer: output_stream,
+ };
+
// save contents
- let mut output = OpenOptions::new()
- .read(true)
- .write(true)
- .open(asset_path)
- .map_err(Error::IoError)?;
- match new_contents.write(&mut output) {
- Ok(_) => Ok(()),
- Err(e) => Err(Error::IoError(e)),
- }
+ new_contents
+ .write(&mut writer)
+ .map_err(|_e| Error::EmbeddingError)?;
+ Ok(())
}
- fn get_object_locations(
+ fn get_object_locations_from_stream(
&self,
- asset_path: &std::path::Path,
+ input_stream: &mut dyn CAIRead,
) -> Result<Vec<HashObjectPositions>> {
- add_required_chunks(asset_path)?;
+ let output_buf: Vec<u8> = Vec::new();
+ let mut output_stream = Cursor::new(output_buf);
- let mut f = std::fs::File::open(asset_path).map_err(|_err| Error::EmbeddingError)?;
+ add_required_chunks(&self.riff_format, input_stream, &mut output_stream)?;
let mut positions: Vec<HashObjectPositions> = Vec::new();
- let (manifest_pos, manifest_len) = get_manifest_pos(&mut f).ok_or(Error::EmbeddingError)?;
+ let (manifest_pos, manifest_len) =
+ get_manifest_pos(&mut output_stream).ok_or(Error::EmbeddingError)?;
positions.push(HashObjectPositions {
offset: usize::value_from(manifest_pos)
@@ -260,7 +476,7 @@ impl AssetIO for RiffIO {
.map_err(|_err| Error::InvalidAsset("value out of range".to_string()))?
+ u64::value_from(manifest_len)
.map_err(|_err| Error::InvalidAsset("value out of range".to_string()))?;
- let file_end = f.metadata()?.len();
+ let file_end = output_stream.seek(SeekFrom::End(0))?;
positions.push(HashObjectPositions {
offset: usize::value_from(end)
.map_err(|_err| Error::InvalidAsset("value out of range".to_string()))?, // len of cai
@@ -272,16 +488,12 @@ impl AssetIO for RiffIO {
Ok(positions)
}
- fn remove_cai_store(&self, asset_path: &Path) -> Result<()> {
- self.save_cai_store(asset_path, &[])
- }
-
- fn remote_ref_writer_ref(&self) -> Option<&dyn RemoteRefEmbed> {
- Some(self)
- }
-
- fn supported_types(&self) -> &[&str] {
- &SUPPORTED_TYPES
+ fn remove_cai_store_from_stream(
+ &self,
+ input_stream: &mut dyn CAIRead,
+ output_stream: &mut dyn CAIReadWrite,
+ ) -> Result<()> {
+ self.write_cai(input_stream, output_stream, &[])
}
}
@@ -315,21 +527,109 @@ impl RemoteRefEmbed for RiffIO {
asset_path: &Path,
embed_ref: crate::asset_io::RemoteRefEmbedType,
) -> Result<()> {
+ let mut input_stream = File::open(asset_path)?;
+
+ let mut output_stream = std::fs::OpenOptions::new()
+ .read(true)
+ .write(true)
+ .open(asset_path)
+ .map_err(Error::IoError)?;
+
+ self.embed_reference_to_stream(&mut input_stream, &mut output_stream, embed_ref)
+ }
+
+ fn embed_reference_to_stream(
+ &self,
+ input_stream: &mut dyn CAIRead,
+ output_stream: &mut dyn CAIReadWrite,
+ embed_ref: RemoteRefEmbedType,
+ ) -> Result<()> {
match embed_ref {
crate::asset_io::RemoteRefEmbedType::Xmp(manifest_uri) => {
- #[cfg(feature = "xmp_write")]
- {
- match self.riff_format.as_ref() {
- "avi" | "wav" => {
- crate::embedded_xmp::add_manifest_uri_to_file(asset_path, &manifest_uri)
+ match self.riff_format.as_ref() {
+ "avi" | "wav" | "webp" => {
+ if let Some(curr_xmp) = self.read_xmp(input_stream) {
+ let mut new_xmp = add_provenance(&curr_xmp, &manifest_uri)?;
+ if new_xmp.len() % 2 == 1 {
+ // pad if needed to even length
+ new_xmp.push(' ');
+ }
+
+ let top_level_chunks = {
+ let mut reader = CAIReadWrapper {
+ reader: input_stream,
+ };
+ Chunk::read(&mut reader, 0)?
+ };
+
+ if top_level_chunks.id() != RIFF_ID {
+ return Err(Error::InvalidAsset("Invalid RIFF format".to_string()));
+ }
+
+ let mut reader = CAIReadWrapper {
+ reader: input_stream,
+ };
+
+ // replace/add manifest in memory
+ let new_contents = inject_c2pa(
+ &top_level_chunks,
+ &mut reader,
+ &[],
+ Some(new_xmp.as_bytes()),
+ &self.riff_format,
+ )?;
+
+ // save contents
+ let mut writer = CAIReadWriteWrapper {
+ reader_writer: output_stream,
+ };
+ new_contents
+ .write(&mut writer)
+ .map_err(|_e| Error::EmbeddingError)?;
+ Ok(())
+ } else {
+ let mut new_xmp = add_provenance(MIN_XMP, &manifest_uri)?;
+
+ if new_xmp.len() % 2 == 1 {
+ // pad if needed to even length
+ new_xmp.push(' ');
+ }
+
+ let top_level_chunks = {
+ let mut reader = CAIReadWrapper {
+ reader: input_stream,
+ };
+ Chunk::read(&mut reader, 0)?
+ };
+
+ if top_level_chunks.id() != RIFF_ID {
+ return Err(Error::InvalidAsset("Invalid RIFF format".to_string()));
+ }
+
+ let mut reader = CAIReadWrapper {
+ reader: input_stream,
+ };
+
+ // replace/add manifest in memory
+ let new_contents = inject_c2pa(
+ &top_level_chunks,
+ &mut reader,
+ &[],
+ Some(new_xmp.as_bytes()),
+ &self.riff_format,
+ )?;
+
+ // save contents
+ let mut writer = CAIReadWriteWrapper {
+ reader_writer: output_stream,
+ };
+ new_contents
+ .write(&mut writer)
+ .map_err(|_e| Error::EmbeddingError)?;
+ Ok(())
}
- _ => Err(Error::XmpNotSupported),
}
- }
-
- #[cfg(not(feature = "xmp_write"))]
- {
- Err(crate::error::Error::MissingFeature("xmp_write".to_string()))
+ _ => Err(Error::UnsupportedType),
}
}
crate::asset_io::RemoteRefEmbedType::StegoS(_) => Err(Error::UnsupportedType),
@@ -337,15 +637,6 @@ impl RemoteRefEmbed for RiffIO {
crate::asset_io::RemoteRefEmbedType::Watermark(_) => Err(Error::UnsupportedType),
}
}
-
- fn embed_reference_to_stream(
- &self,
- _source_stream: &mut dyn CAIRead,
- _output_stream: &mut dyn CAIReadWrite,
- _embed_ref: RemoteRefEmbedType,
- ) -> Result<()> {
- Err(Error::UnsupportedType)
- }
}
#[cfg(test)]
@@ -360,6 +651,7 @@ pub mod tests {
use crate::utils::{
hash_utils::vec_compare,
test::{fixture_path, temp_dir_path},
+ xmp_inmemory_utils::extract_provenance,
};
#[test]
@@ -434,4 +726,123 @@ pub mod tests {
_ => unreachable!(),
}
}
+
+ #[test]
+ fn test_read_xmp() {
+ let source = fixture_path("test_xmp.webp");
+ let mut reader = std::fs::File::open(source).unwrap();
+
+ let riff_io = RiffIO::new("webp");
+
+ let xmp = riff_io.read_xmp(&mut reader).unwrap();
+ println!("XMP: {xmp}");
+ }
+
+ #[test]
+ fn test_write_xmp() {
+ let more_data = "some more test data";
+ let source = fixture_path("test_xmp.webp");
+
+ let mut success = false;
+ if let Ok(temp_dir) = tempdir() {
+ let output = temp_dir_path(&temp_dir, "test_xmp.webp");
+
+ std::fs::copy(source, &output).unwrap();
+
+ let riff_io = RiffIO::new("webp");
+
+ if let Some(embed_handler) = riff_io.remote_ref_writer_ref() {
+ if let Ok(()) = embed_handler.embed_reference(
+ output.as_path(),
+ RemoteRefEmbedType::Xmp(more_data.to_string()),
+ ) {
+ let mut output_stream = std::fs::File::open(&output).unwrap();
+
+ // check the xmp
+ if let Some(xmp) = riff_io.read_xmp(&mut output_stream) {
+ println!("XMP: {xmp}");
+
+ if let Some(xmp_val) = extract_provenance(&xmp) {
+ if xmp_val == more_data {
+ success = true;
+ }
+ }
+ }
+ }
+ }
+ }
+ assert!(success)
+ }
+
+ #[test]
+ fn test_insert_xmp() {
+ let more_data = "some more test data";
+ let source = fixture_path("test.webp");
+
+ let mut success = false;
+ if let Ok(temp_dir) = tempdir() {
+ let output = temp_dir_path(&temp_dir, "test.webp");
+
+ std::fs::copy(source, &output).unwrap();
+
+ let riff_io = RiffIO::new("webp");
+
+ if let Some(embed_handler) = riff_io.remote_ref_writer_ref() {
+ if let Ok(()) = embed_handler.embed_reference(
+ output.as_path(),
+ RemoteRefEmbedType::Xmp(more_data.to_string()),
+ ) {
+ let mut output_stream = std::fs::File::open(&output).unwrap();
+
+ // check the xmp
+ if let Some(xmp) = riff_io.read_xmp(&mut output_stream) {
+ println!("XMP: {xmp}");
+
+ if let Some(xmp_val) = extract_provenance(&xmp) {
+ if xmp_val == more_data {
+ success = true;
+ }
+ }
+ }
+ }
+ }
+ }
+ assert!(success)
+ }
+
+ #[test]
+ fn test_insert_xmp_lossless() {
+ let more_data = "some more test data";
+ let source = fixture_path("test_lossless.webp");
+
+ let mut success = false;
+ if let Ok(temp_dir) = tempdir() {
+ let output = temp_dir_path(&temp_dir, "test_lossless.webp");
+
+ std::fs::copy(source, &output).unwrap();
+
+ let riff_io = RiffIO::new("webp");
+
+ if let Some(embed_handler) = riff_io.remote_ref_writer_ref() {
+ if let Ok(()) = embed_handler.embed_reference(
+ output.as_path(),
+ RemoteRefEmbedType::Xmp(more_data.to_string()),
+ ) {
+ let mut output_stream = std::fs::File::open(&output).unwrap();
+
+ // check the xmp
+ if let Some(xmp) = riff_io.read_xmp(&mut output_stream) {
+ println!("XMP: {xmp}");
+
+ if let Some(xmp_val) = extract_provenance(&xmp) {
+ if xmp_val == more_data {
+ success = true;
+ }
+ }
+ }
+ }
+ }
+ }
+ assert!(success)
+ }
}
diff --git a/sdk/src/asset_io.rs b/sdk/src/asset_io.rs
@@ -48,6 +48,27 @@ impl CAIRead for std::io::Cursor<&mut [u8]> {}
impl CAIRead for std::io::Cursor<Vec<u8>> {}
impl CAIRead for NamedTempFile {}
+// Helper struct to create a concrete type for CAIRead when
+// that is required. For example a function defined like this
+// pub fn read<T>(&self, reader: &mut T) cannot currently accept
+// a CAIRead trait because it is not Sized (bound to a object).
+// This will likely change in a future version of Rust.
+pub(crate) struct CAIReadWrapper<'a> {
+ pub reader: &'a mut dyn CAIRead,
+}
+
+impl Read for CAIReadWrapper<'_> {
+ fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+ self.reader.read(buf)
+ }
+}
+
+impl Seek for CAIReadWrapper<'_> {
+ fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result<u64> {
+ self.reader.seek(pos)
+ }
+}
+
pub trait CAIReadWrite: CAIRead + Write {}
impl CAIReadWrite for std::fs::File {}
@@ -55,6 +76,38 @@ impl CAIReadWrite for std::io::Cursor<&mut [u8]> {}
impl CAIReadWrite for std::io::Cursor<Vec<u8>> {}
impl CAIReadWrite for NamedTempFile {}
+// Helper struct to create a concrete type for CAIReadWrite when
+// that is required. For example a function defined like this
+// pub fn write<T>(&self, writer: &mut T) cannot currently accept
+// a CAIReadWrite trait because it is not Sized (bound to a object).
+// This will likely change in a future version of Rust.
+// go away in future revisions of Rust.
+pub(crate) struct CAIReadWriteWrapper<'a> {
+ pub reader_writer: &'a mut dyn CAIReadWrite,
+}
+
+impl Read for CAIReadWriteWrapper<'_> {
+ fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+ self.reader_writer.read(buf)
+ }
+}
+
+impl Write for CAIReadWriteWrapper<'_> {
+ fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+ self.reader_writer.write(buf)
+ }
+
+ fn flush(&mut self) -> std::io::Result<()> {
+ self.reader_writer.flush()
+ }
+}
+
+impl Seek for CAIReadWriteWrapper<'_> {
+ fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result<u64> {
+ self.reader_writer.seek(pos)
+ }
+}
+
// Interface for in memory CAI reading
pub trait CAIReader: Sync + Send {
// Return entire CAI block as Vec<u8>
diff --git a/sdk/src/ingredient.rs b/sdk/src/ingredient.rs
@@ -1306,7 +1306,7 @@ mod tests_file_io {
let ingredient = Ingredient::from_file(ap).expect("from_file");
stats(&ingredient);
- println!("ingredient = {}", ingredient);
+ println!("ingredient = {ingredient}");
assert_eq!(ingredient.title(), BAD_SIGNATURE_JPEG);
assert_eq!(ingredient.format(), "image/jpeg");
test_thumbnail(&ingredient, "image/jpeg");
diff --git a/sdk/src/jumbf_io.rs b/sdk/src/jumbf_io.rs
@@ -358,7 +358,6 @@ pub mod tests {
let handlers: Vec<Box<dyn AssetIO>> = vec![
Box::new(C2paIO::new("")),
Box::new(BmffIO::new("")),
- Box::new(RiffIO::new("")),
Box::new(TiffIO::new("")),
];
diff --git a/sdk/src/store.rs b/sdk/src/store.rs
@@ -1381,19 +1381,21 @@ impl Store {
}
}
- if block_end as u64 > stream_len {
- return Err(Error::BadParam(
- "data hash exclusions out of range".to_string(),
- ));
- }
-
if found_jumbf {
// add exclusion hash for bytes before and after jumbf
let mut dh = DataHash::new("jumbf manifest", alg, None);
if block_end > block_start {
dh.add_exclusion(Exclusion::new(block_start, block_end - block_start));
}
+
if calc_hashes {
+ // this check is only valid on the final sized asset
+ if block_end as u64 > stream_len {
+ return Err(Error::BadParam(
+ "data hash exclusions out of range".to_string(),
+ ));
+ }
+
dh.gen_hash_from_stream(stream)?;
} else {
match alg {
diff --git a/sdk/src/utils/xmp_inmemory_utils.rs b/sdk/src/utils/xmp_inmemory_utils.rs
@@ -26,6 +26,8 @@ use crate::{
const RDF_DESCRIPTION: &[u8] = b"rdf:Description";
+pub const MIN_XMP: &str = r#"<?xpacket begin="" id="W5M0MpCehiHzreSzNTczkc9d"?><x:xmpmeta xmlns:x="adobe:ns:meta/" x:xmptk="XMP Core 6.0.0"><rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><rdf:Description rdf:about="" > </rdf:Description></rdf:RDF> </x:xmpmeta> "#;
+
#[derive(Default)]
pub struct XmpInfo {
pub document_id: Option<String>,
@@ -160,8 +162,7 @@ fn extract_document_id(xmp: &str) -> Option<String> {
}
/// add or replace a dc:provenance value to xmp, including dc:terms if needed
-#[allow(dead_code)] // keep for future
-fn add_provenance(xmp: &str, provenance: &str) -> Result<String> {
+pub fn add_provenance(xmp: &str, provenance: &str) -> Result<String> {
let xmp = add_xmp_key(xmp, "xmlns:dcterms", "http://purl.org/dc/terms/")?;
add_xmp_key(&xmp, "dcterms:provenance", provenance)
}
@@ -190,12 +191,6 @@ mod tests {
</rdf:RDF>
</x:xmpmeta>"#;
- const MIN_XMP: &str = r#"<?xpacket begin="" id="W5M0MpCehiHzreSzNTczkc9d"?>
- <x:xmpmeta xmlns:x="adobe:ns:meta/" x:xmptk="XMP Core 6.0.0">
- <rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
- <rdf:Description rdf:about="" > </rdf:Description>
- </rdf:RDF> </x:xmpmeta> "#;
-
const PROVENANCE: &str =
"self#jumbf=c2pa/contentauth:urn:uuid:a58065fb-79ae-4eb3-87b9-a19830860059/c2pa.claim";
diff --git a/sdk/tests/fixtures/test.webp b/sdk/tests/fixtures/test.webp
Binary files differ.
diff --git a/sdk/tests/fixtures/test_lossless.webp b/sdk/tests/fixtures/test_lossless.webp
Binary files differ.
diff --git a/sdk/tests/fixtures/test_xmp.webp b/sdk/tests/fixtures/test_xmp.webp
Binary files differ.
