commit 123e4de2829ab15fdb2965e5e141e008fff52577
parent 7bbf729e4f0c33c45b6d535567ebfa7413d987f0
Author: mauricefisher64 <92736594+mauricefisher64@users.noreply.github.com>
Date: Fri, 23 Feb 2024 14:07:13 -0500
Support streaming writes for TIFF (#410)
* TIFF read support
* Start of alternate apporach
* simplify experiment
* More intermeadiate work
* Working support for stripped TIFFs
* More TIFF enhnacements
* SubFile stuff
* DNG push
* DNG bug fixes
* Endian fixes
* Support for GPS IFD
Clippy fixes
* Patch support
bug fixes
* Code cleanup
* Review changes
* clippy fmt fix
* wasm fix
* Fix for linux build
* Address PR comments
* fix up comments
* TIFF streaming write support
Build fixes
---------
Co-authored-by: mauricefisher64 <maurice_fisher@hotmail.com>
Diffstat:
5 files changed, 168 insertions(+), 73 deletions(-)
diff --git a/sdk/src/assertions/user_cbor.rs b/sdk/src/assertions/user_cbor.rs
@@ -75,7 +75,6 @@ pub mod tests {
#![allow(clippy::unwrap_used)]
use super::*;
-
const LABEL: &str = "user_test_assertion";
const DATA: &str = r#"{ "l1":"some data", "l2":"some other data" }"#;
diff --git a/sdk/src/asset_handlers/jpeg_io.rs b/sdk/src/asset_handlers/jpeg_io.rs
@@ -1036,7 +1036,6 @@ pub mod tests {
use std::io::{Read, Seek};
use super::*;
-
#[test]
fn test_extract_xmp() {
let contents = Bytes::from_static(b"http://ns.adobe.com/xap/1.0/\0stuff");
diff --git a/sdk/src/asset_handlers/tiff_io.rs b/sdk/src/asset_handlers/tiff_io.rs
@@ -16,6 +16,7 @@ use std::{
fs::OpenOptions,
io::{Cursor, Read, Seek, SeekFrom, Write},
path::Path,
+ vec,
};
use atree::{Arena, Token};
@@ -26,8 +27,9 @@ use tempfile::Builder;
use crate::{
asset_io::{
- rename_or_copy, AssetIO, AssetPatch, CAIRead, CAIReadWrite, CAIReader, ComposedManifestRef,
- HashBlockObjectType, HashObjectPositions, RemoteRefEmbed, RemoteRefEmbedType,
+ rename_or_copy, AssetIO, AssetPatch, CAIRead, CAIReadWrite, CAIReader, CAIWriter,
+ ComposedManifestRef, HashBlockObjectType, HashObjectPositions, RemoteRefEmbed,
+ RemoteRefEmbedType,
},
error::{Error, Result},
};
@@ -349,6 +351,16 @@ fn decode_offset(offset_file_native: u64, endianness: Endianness, big_tiff: bool
Ok(offset)
}
+fn stream_len(reader: &mut dyn CAIRead) -> crate::Result<u64> {
+ let old_pos = reader.stream_position()?;
+ let len = reader.seek(SeekFrom::End(0))?;
+
+ if old_pos != len {
+ reader.seek(SeekFrom::Start(old_pos))?;
+ }
+
+ Ok(len)
+}
// create tree of TIFF structure IFDs and IFD entries.
fn map_tiff<R: ?Sized>(
input: &mut R,
@@ -648,7 +660,7 @@ impl<T: Read + Write + Seek> TiffCloner<T> {
self.additional_ifds.insert(entry.entry_tag, entry);
}
- fn clone_image_data<R: Read + Seek>(
+ fn clone_image_data<R: Read + Seek + ?Sized>(
&mut self,
target_ifd: &mut BTreeMap<u16, IfdClonedEntry>,
asset_reader: &mut R,
@@ -871,7 +883,7 @@ impl<T: Read + Write + Seek> TiffCloner<T> {
Ok(())
}
- fn clone_sub_files<R: Read + Seek>(
+ fn clone_sub_files<R: Read + Seek + ?Sized>(
&mut self,
tiff_tree: &Arena<ImageFileDirectory>,
page: Token,
@@ -920,7 +932,7 @@ impl<T: Read + Write + Seek> TiffCloner<T> {
Ok(offset_map)
}
- pub fn clone_tiff<R: Read + Seek>(
+ pub fn clone_tiff<R: Read + Seek + ?Sized>(
&mut self,
tiff_tree: &mut Arena<ImageFileDirectory>,
page_0: Token,
@@ -1004,7 +1016,7 @@ impl<T: Read + Write + Seek> TiffCloner<T> {
Ok(())
}
- fn clone_ifd_entries<R: Read + Seek>(
+ fn clone_ifd_entries<R: Read + Seek + ?Sized>(
&mut self,
entries: &HashMap<u16, IfdEntry>,
asset_reader: &mut R,
@@ -1271,7 +1283,7 @@ impl<T: Read + Write + Seek> TiffCloner<T> {
}
}
-fn tiff_clone_with_tags<R: Read + Seek, W: Read + Write + Seek>(
+fn tiff_clone_with_tags<R: Read + Seek + ?Sized, W: Read + Write + Seek + ?Sized>(
writer: &mut W,
asset_reader: &mut R,
tiff_tags: Vec<IfdClonedEntry>,
@@ -1290,17 +1302,25 @@ fn tiff_clone_with_tags<R: Read + Seek, W: Read + Write + Seek>(
Ok(())
}
-
-fn add_required_tags(asset_path: &std::path::Path) -> Result<()> {
- let mut f = std::fs::File::open(asset_path)?;
+fn add_required_tags_to_stream(
+ input_stream: &mut dyn CAIRead,
+ output_stream: &mut dyn CAIReadWrite,
+) -> Result<()> {
let tiff_io = TiffIO {};
- match tiff_io.read_cai(&mut f) {
- Ok(_) => Ok(()),
+ match tiff_io.read_cai(input_stream) {
+ Ok(_) => {
+ // just clone
+ input_stream.rewind()?;
+ output_stream.rewind()?;
+ std::io::copy(input_stream, output_stream)?;
+ Ok(())
+ }
Err(Error::JumbfNotFound) => {
// allocate enough bytes so that value is not stored in offset field
let some_bytes = vec![0u8; 10];
- tiff_io.save_cai_store(asset_path, &some_bytes)
+ let tio = TiffIO {};
+ tio.write_cai(input_stream, output_stream, &some_bytes)
}
Err(e) => Err(e),
}
@@ -1389,13 +1409,85 @@ impl AssetIO for TiffIO {
}
fn save_cai_store(&self, asset_path: &std::path::Path, store_bytes: &[u8]) -> Result<()> {
+ let mut input_stream = std::fs::OpenOptions::new()
+ .read(true)
+ .open(asset_path)
+ .map_err(Error::IoError)?;
+
let mut temp_file = Builder::new()
.prefix("c2pa_temp")
.rand_bytes(5)
.tempfile()?;
- let mut reader = std::fs::File::open(asset_path)?;
+ self.write_cai(&mut input_stream, &mut temp_file, store_bytes)?;
+
+ // copy temp file to asset
+ rename_or_copy(temp_file, asset_path)
+ }
+
+ fn get_object_locations(
+ &self,
+ asset_path: &std::path::Path,
+ ) -> Result<Vec<crate::asset_io::HashObjectPositions>> {
+ let mut input_stream =
+ std::fs::File::open(asset_path).map_err(|_err| Error::EmbeddingError)?;
+
+ self.get_object_locations_from_stream(&mut input_stream)
+ }
+
+ fn remove_cai_store(&self, asset_path: &std::path::Path) -> Result<()> {
+ let mut input_file = std::fs::File::open(asset_path)?;
+
+ let mut temp_file = Builder::new()
+ .prefix("c2pa_temp")
+ .rand_bytes(5)
+ .tempfile()?;
+
+ self.remove_cai_store_from_stream(&mut input_file, &mut temp_file)?;
+
+ // copy temp file to asset
+ rename_or_copy(temp_file, asset_path)
+ }
+ fn new(_asset_type: &str) -> Self
+ where
+ Self: Sized,
+ {
+ TiffIO {}
+ }
+
+ fn get_handler(&self, asset_type: &str) -> Box<dyn AssetIO> {
+ Box::new(TiffIO::new(asset_type))
+ }
+
+ fn get_reader(&self) -> &dyn CAIReader {
+ self
+ }
+
+ fn get_writer(&self, asset_type: &str) -> Option<Box<dyn CAIWriter>> {
+ Some(Box::new(TiffIO::new(asset_type)))
+ }
+
+ fn remote_ref_writer_ref(&self) -> Option<&dyn RemoteRefEmbed> {
+ Some(self)
+ }
+
+ fn composed_data_ref(&self) -> Option<&dyn ComposedManifestRef> {
+ Some(self)
+ }
+
+ fn supported_types(&self) -> &[&str] {
+ &SUPPORTED_TYPES
+ }
+}
+
+impl CAIWriter for TiffIO {
+ fn write_cai(
+ &self,
+ input_stream: &mut dyn CAIRead,
+ output_stream: &mut dyn CAIReadWrite,
+ store_bytes: &[u8],
+ ) -> Result<()> {
let l = u64::value_from(store_bytes.len())
.map_err(|_err| Error::InvalidAsset("value out of range".to_string()))?;
@@ -1406,21 +1498,24 @@ impl AssetIO for TiffIO {
value_bytes: store_bytes.to_vec(),
};
- tiff_clone_with_tags(&mut temp_file, &mut reader, vec![entry])?;
-
- // copy temp file to asset
- rename_or_copy(temp_file, asset_path)
+ tiff_clone_with_tags(output_stream, input_stream, vec![entry])
}
- fn get_object_locations(
+ fn get_object_locations_from_stream(
&self,
- asset_path: &std::path::Path,
- ) -> Result<Vec<crate::asset_io::HashObjectPositions>> {
- add_required_tags(asset_path)?;
+ input_stream: &mut dyn CAIRead,
+ ) -> Result<Vec<HashObjectPositions>> {
+ let len = stream_len(input_stream)?;
+ let vec_cap = usize::value_from(len)
+ .map_err(|_err| Error::InvalidAsset("value out of range".to_owned()))?;
+ let output_buf: Vec<u8> = Vec::with_capacity(vec_cap + 100);
+
+ let mut output_stream = Cursor::new(output_buf);
- let mut asset_reader = std::fs::File::open(asset_path)?;
+ add_required_tags_to_stream(input_stream, &mut output_stream)?;
+ output_stream.rewind()?;
- let (idfs, first_idf_token, e, big_tiff) = map_tiff(&mut asset_reader)?;
+ let (idfs, first_idf_token, e, big_tiff) = map_tiff(&mut output_stream)?;
let cai_ifd_entry = match idfs[first_idf_token].data.get_tag(C2PA_TAG) {
Some(ifd) => ifd,
@@ -1447,56 +1542,24 @@ impl AssetIO for TiffIO {
}])
}
- fn remove_cai_store(&self, asset_path: &std::path::Path) -> Result<()> {
- let mut temp_file = Builder::new()
- .prefix("c2pa_temp")
- .rand_bytes(5)
- .tempfile()?;
-
- let mut asset_reader = std::fs::File::open(asset_path)?;
-
- let (mut idfs, page_0, e, big_tiff) = map_tiff(&mut asset_reader)?;
+ fn remove_cai_store_from_stream(
+ &self,
+ input_stream: &mut dyn CAIRead,
+ output_stream: &mut dyn CAIReadWrite,
+ ) -> Result<()> {
+ let (mut idfs, page_0, e, big_tiff) = map_tiff(input_stream)?;
- let mut bo = ByteOrdered::new(&mut temp_file, e);
+ let mut bo = ByteOrdered::new(output_stream, e);
let mut tc = TiffCloner::new(e, big_tiff, &mut bo)?;
match idfs[page_0].data.entries.remove(&C2PA_TAG) {
Some(_ifd) => {
- tc.clone_tiff(&mut idfs, page_0, &mut asset_reader)?;
-
- // copy temp file to asset
- rename_or_copy(temp_file, asset_path)
+ tc.clone_tiff(&mut idfs, page_0, input_stream)?;
+ Ok(())
}
None => Ok(()),
}
}
-
- fn new(_asset_type: &str) -> Self
- where
- Self: Sized,
- {
- TiffIO {}
- }
-
- fn get_handler(&self, asset_type: &str) -> Box<dyn AssetIO> {
- Box::new(TiffIO::new(asset_type))
- }
-
- fn get_reader(&self) -> &dyn CAIReader {
- self
- }
-
- fn remote_ref_writer_ref(&self) -> Option<&dyn RemoteRefEmbed> {
- Some(self)
- }
-
- fn composed_data_ref(&self) -> Option<&dyn ComposedManifestRef> {
- Some(self)
- }
-
- fn supported_types(&self) -> &[&str] {
- &SUPPORTED_TYPES
- }
}
impl AssetPatch for TiffIO {
@@ -1643,6 +1706,44 @@ pub mod tests {
}
}
+ #[test]
+ fn test_get_object_location() {
+ let data = "some data";
+
+ let source = crate::utils::test::fixture_path("TUSCANY.TIF");
+
+ let temp_dir = tempdir().unwrap();
+ let output = temp_dir_path(&temp_dir, "test.tif");
+
+ std::fs::copy(source, &output).unwrap();
+
+ let tiff_io = TiffIO {};
+
+ // save data to tiff
+ tiff_io.save_cai_store(&output, data.as_bytes()).unwrap();
+
+ // read data back
+ let loaded = tiff_io.read_cai_store(&output).unwrap();
+
+ assert_eq!(&loaded, data.as_bytes());
+
+ let mut success = false;
+ if let Ok(locations) = tiff_io.get_object_locations(&output) {
+ for op in locations {
+ if op.htype == HashBlockObjectType::Cai {
+ let mut of = std::fs::File::open(&output).unwrap();
+
+ let mut manifests_buf: Vec<u8> = vec![0u8; op.length];
+ of.seek(SeekFrom::Start(op.offset as u64)).unwrap();
+ of.read_exact(manifests_buf.as_mut_slice()).unwrap();
+ if crate::hash_utils::vec_compare(&manifests_buf, data.as_bytes()) {
+ success = true;
+ }
+ }
+ }
+ }
+ assert!(success);
+ }
/* disable until I find smaller DNG
#[test]
fn test_read_write_dng_manifest() {
diff --git a/sdk/src/ingredient.rs b/sdk/src/ingredient.rs
@@ -1338,7 +1338,6 @@ mod tests {
use wasm_bindgen_test::*;
use super::*;
-
#[cfg(target_arch = "wasm32")]
wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser);
diff --git a/sdk/src/jumbf_io.rs b/sdk/src/jumbf_io.rs
@@ -377,11 +377,8 @@ pub mod tests {
#[test]
fn test_no_writer() {
- let handlers: Vec<Box<dyn AssetIO>> = vec![
- Box::new(C2paIO::new("")),
- Box::new(BmffIO::new("")),
- Box::new(TiffIO::new("")),
- ];
+ let handlers: Vec<Box<dyn AssetIO>> =
+ vec![Box::new(C2paIO::new("")), Box::new(BmffIO::new(""))];
// build handler map
for h in handlers {
