From d3653224b6115eb55667e0bb2feaa2a4fc2436b0 Mon Sep 17 00:00:00 2001 From: AusAgentSmith Date: Tue, 7 Jul 2026 22:57:24 +0000 Subject: [PATCH] fix: preserve PAR2 file order and count all recovery slices --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/packets.rs | 124 +++++++++++++++++++++++++----- src/repair.rs | 202 ++++++++++++++++++++++++++++++++++++++++++++++++- src/types.rs | 4 + 5 files changed, 309 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 51fc2a4..747c88a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -351,7 +351,7 @@ dependencies = [ [[package]] name = "rust-par2" -version = "0.1.2" +version = "0.1.3" dependencies = [ "crc32fast", "md-5", diff --git a/Cargo.toml b/Cargo.toml index 5721657..863ef4d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rust-par2" -version = "0.1.2" +version = "0.1.3" edition = "2024" description = "Pure Rust PAR2 verify and repair with SIMD-accelerated Galois field arithmetic" license = "MIT OR Apache-2.0" diff --git a/src/packets.rs b/src/packets.rs index 208acea..a21de87 100644 --- a/src/packets.rs +++ b/src/packets.rs @@ -56,6 +56,7 @@ struct ParseState { recovery_set_id: Option, slice_size: Option, nr_files: Option, + file_order: Vec, /// FileDesc data keyed by File ID. file_descs: HashMap, /// IFSC (slice checksum) data keyed by File ID. @@ -95,6 +96,7 @@ pub fn parse_par2_reader( recovery_set_id: None, slice_size: None, nr_files: None, + file_order: Vec::new(), file_descs: HashMap::new(), ifsc_data: HashMap::new(), recovery_count: 0, @@ -207,24 +209,6 @@ pub fn parse_par2_reader( } packets_parsed += 1; - - // Early exit optimisation: once we have all file descs and IFSCs, we - // can stop (avoids reading huge recovery volumes in concatenated files). - if let Some(nr) = state.nr_files { - if state.file_descs.len() == nr as usize - && state.ifsc_data.len() == nr as usize - && state.slice_size.is_some() - { - // If the file is large, stop early like SABnzbd does. - if file_size > 10 * 1024 * 1024 { - debug!( - packets_parsed, - "parsed all file metadata, stopping early on large file" - ); - break; - } - } - } } if packets_parsed == 0 { @@ -262,6 +246,7 @@ pub fn parse_par2_reader( Ok(Par2FileSet { recovery_set_id, slice_size, + file_order: state.file_order, files, recovery_block_count: state.recovery_count, creator: state.creator, @@ -391,11 +376,26 @@ fn parse_main(data: &[u8], state: &mut ParseState) { let slice_size = u64::from_le_bytes(data[32..40].try_into().unwrap()); let nr_files = u32::from_le_bytes(data[40..44].try_into().unwrap()); + let ids = &data[44..]; + let count = (nr_files as usize).min(ids.len() / 16); + let mut file_order = Vec::with_capacity(count); + for i in 0..count { + let start = i * 16; + let mut id = [0u8; 16]; + id.copy_from_slice(&ids[start..start + 16]); + file_order.push(id); + } - trace!(slice_size, nr_files, "parsed Main"); + trace!( + slice_size, + nr_files, + main_file_ids = file_order.len(), + "parsed Main" + ); state.slice_size = Some(slice_size); state.nr_files = Some(nr_files); + state.file_order = file_order; } /// Parse a Creator packet. @@ -448,6 +448,21 @@ fn scan_for_magic(reader: &mut R, file_size: u64) -> io::Result< #[cfg(test)] mod tests { use super::*; + use std::io::Cursor; + + fn build_packet(set_id: Id16, packet_type: &[u8; 16], body: &[u8]) -> Vec { + let mut data = Vec::with_capacity(32 + body.len()); + data.extend_from_slice(&set_id); + data.extend_from_slice(packet_type); + data.extend_from_slice(body); + + let mut packet = Vec::with_capacity(32 + data.len()); + packet.extend_from_slice(PAR2_MAGIC); + packet.extend_from_slice(&((HEADER_SIZE + body.len()) as u64).to_le_bytes()); + packet.extend_from_slice(&Md5::digest(&data)); + packet.extend_from_slice(&data); + packet + } /// Test parsing the real PAR2 file from SABnzbd test data. #[test] @@ -548,4 +563,75 @@ mod tests { "recovery volume should have at least 1 recovery block" ); } + + #[test] + fn test_parse_main_preserves_file_order() { + let set_id = [0xAB; 16]; + let file_a = [0x10; 16]; + let file_b = [0x01; 16]; + + let mut main_body = Vec::new(); + main_body.extend_from_slice(&4096u64.to_le_bytes()); + main_body.extend_from_slice(&2u32.to_le_bytes()); + main_body.extend_from_slice(&file_a); + main_body.extend_from_slice(&file_b); + + let packet = build_packet(set_id, TYPE_MAIN, &main_body); + + let mut state = ParseState { + recovery_set_id: None, + slice_size: None, + nr_files: None, + file_order: Vec::new(), + file_descs: HashMap::new(), + ifsc_data: HashMap::new(), + recovery_count: 0, + creator: None, + }; + parse_main(&packet[32..], &mut state); + assert_eq!(state.file_order, vec![file_a, file_b]); + } + + #[test] + fn test_large_recovery_volume_counts_all_blocks() { + let set_id = [0x42; 16]; + let file_id = [0x24; 16]; + let mut bytes = Vec::new(); + + let mut main_body = Vec::new(); + main_body.extend_from_slice(&524_288u64.to_le_bytes()); + main_body.extend_from_slice(&1u32.to_le_bytes()); + main_body.extend_from_slice(&file_id); + bytes.extend_from_slice(&build_packet(set_id, TYPE_MAIN, &main_body)); + + let mut file_desc_body = Vec::new(); + file_desc_body.extend_from_slice(&file_id); + file_desc_body.extend_from_slice(&[0x11; 16]); + file_desc_body.extend_from_slice(&[0x22; 16]); + file_desc_body.extend_from_slice(&524_288u64.to_le_bytes()); + file_desc_body.extend_from_slice(b"payload.bin\0"); + while file_desc_body.len() % 4 != 0 { + file_desc_body.push(0); + } + bytes.extend_from_slice(&build_packet(set_id, TYPE_FILE_DESC, &file_desc_body)); + + let mut ifsc_body = Vec::new(); + ifsc_body.extend_from_slice(&file_id); + ifsc_body.extend_from_slice(&[0x33; 16]); + ifsc_body.extend_from_slice(&0u32.to_le_bytes()); + bytes.extend_from_slice(&build_packet(set_id, TYPE_IFSC, &ifsc_body)); + + for exp in 0..20u32 { + let mut rec_body = Vec::with_capacity(4 + 524_288); + rec_body.extend_from_slice(&exp.to_le_bytes()); + rec_body.extend(std::iter::repeat_n(exp as u8, 524_288)); + bytes.extend_from_slice(&build_packet(set_id, TYPE_RECOVERY, &rec_body)); + } + + let mut cursor = Cursor::new(bytes); + let file_size = cursor.get_ref().len() as u64; + let parsed = parse_par2_reader(&mut cursor, file_size).unwrap(); + assert_eq!(parsed.recovery_block_count, 20); + assert_eq!(parsed.file_order, vec![file_id]); + } } diff --git a/src/repair.rs b/src/repair.rs index 9aee7c3..05e7f9a 100644 --- a/src/repair.rs +++ b/src/repair.rs @@ -361,11 +361,19 @@ fn build_block_map(file_set: &Par2FileSet) -> BlockMap { let mut files = Vec::new(); let mut block_offset = 0u32; - // Sort files by file ID for deterministic ordering (same as par2cmdline) - let mut sorted_files: Vec<_> = file_set.files.values().collect(); - sorted_files.sort_by_key(|f| f.file_id); + let ordered_files: Vec<_> = if file_set.file_order.is_empty() { + let mut fallback: Vec<_> = file_set.files.values().collect(); + fallback.sort_by_key(|f| f.file_id); + fallback + } else { + file_set + .file_order + .iter() + .filter_map(|id| file_set.files.get(id)) + .collect() + }; - for f in sorted_files { + for f in ordered_files { let block_count = if slice_size == 0 { 0 } else { @@ -479,6 +487,134 @@ fn read_source_block( #[cfg(test)] mod tests { use super::*; + use std::path::PathBuf; + + use crc32fast::hash as crc32_hash; + use md5::{Digest, Md5}; + + use crate::packets::{HEADER_SIZE, MAGIC}; + + const TYPE_MAIN: &[u8; 16] = b"PAR 2.0\x00Main\x00\x00\x00\x00"; + const TYPE_FILE_DESC: &[u8; 16] = b"PAR 2.0\x00FileDesc"; + const TYPE_IFSC: &[u8; 16] = b"PAR 2.0\x00IFSC\x00\x00\x00\x00"; + const TYPE_RECOVERY: &[u8; 16] = b"PAR 2.0\x00RecvSlic"; + + fn build_packet(set_id: [u8; 16], packet_type: &[u8; 16], body: &[u8]) -> Vec { + let mut data = Vec::with_capacity(32 + body.len()); + data.extend_from_slice(&set_id); + data.extend_from_slice(packet_type); + data.extend_from_slice(body); + + let mut packet = Vec::with_capacity(32 + data.len()); + packet.extend_from_slice(MAGIC); + packet.extend_from_slice(&((HEADER_SIZE + body.len()) as u64).to_le_bytes()); + packet.extend_from_slice(&Md5::digest(&data)); + packet.extend_from_slice(&data); + packet + } + + fn file_md5(data: &[u8]) -> [u8; 16] { + Md5::digest(data).into() + } + + fn hash_16k(data: &[u8]) -> [u8; 16] { + Md5::digest(&data[..data.len().min(16_384)]).into() + } + + fn padded_block(data: &[u8], block_index: usize, slice_size: usize) -> Vec { + let start = block_index * slice_size; + let end = (start + slice_size).min(data.len()); + let mut block = vec![0u8; slice_size]; + if start < data.len() { + block[..end - start].copy_from_slice(&data[start..end]); + } + block + } + + fn write_par2_fixture( + dir: &Path, + slice_size: usize, + files: &[(&str, [u8; 16], Vec)], + main_order: &[[u8; 16]], + recovery_exponents: &[u32], + ) -> PathBuf { + let set_id = [0x5Au8; 16]; + let mut index_bytes = Vec::new(); + + let mut main_body = Vec::new(); + main_body.extend_from_slice(&(slice_size as u64).to_le_bytes()); + main_body.extend_from_slice(&(main_order.len() as u32).to_le_bytes()); + for file_id in main_order { + main_body.extend_from_slice(file_id); + } + index_bytes.extend_from_slice(&build_packet(set_id, TYPE_MAIN, &main_body)); + + for (filename, file_id, data) in files { + std::fs::write(dir.join(filename), data).unwrap(); + + let mut desc_body = Vec::new(); + desc_body.extend_from_slice(file_id); + desc_body.extend_from_slice(&file_md5(data)); + desc_body.extend_from_slice(&hash_16k(data)); + desc_body.extend_from_slice(&(data.len() as u64).to_le_bytes()); + desc_body.extend_from_slice(filename.as_bytes()); + desc_body.push(0); + while desc_body.len() % 4 != 0 { + desc_body.push(0); + } + index_bytes.extend_from_slice(&build_packet(set_id, TYPE_FILE_DESC, &desc_body)); + + let block_count = data.len().div_ceil(slice_size); + let mut ifsc_body = Vec::new(); + ifsc_body.extend_from_slice(file_id); + for block_idx in 0..block_count { + let block = padded_block(data, block_idx, slice_size); + ifsc_body.extend_from_slice(&file_md5(&block)); + ifsc_body.extend_from_slice(&crc32_hash(&block).to_le_bytes()); + } + index_bytes.extend_from_slice(&build_packet(set_id, TYPE_IFSC, &ifsc_body)); + } + + std::fs::write(dir.join("fixture.par2"), &index_bytes).unwrap(); + + let ordered_files: Vec<_> = main_order + .iter() + .map(|wanted| { + files + .iter() + .find(|(_, file_id, _)| file_id == wanted) + .unwrap() + }) + .collect(); + let input_blocks: Vec> = ordered_files + .iter() + .flat_map(|(_, _, data)| { + let count = data.len().div_ceil(slice_size); + (0..count).map(move |block_idx| padded_block(data, block_idx, slice_size)) + }) + .collect(); + + let input_count = input_blocks.len(); + let enc = GfMatrix::par2_encoding_matrix(input_count, recovery_exponents); + let srcs: Vec<&[u8]> = input_blocks.iter().map(Vec::as_slice).collect(); + let mut vol_bytes = index_bytes.clone(); + + for (row_idx, &exp) in recovery_exponents.iter().enumerate() { + let coeffs: Vec = (0..input_count) + .map(|col| enc.get(input_count + row_idx, col)) + .collect(); + let mut recovery = vec![0u8; slice_size]; + gf_simd::mul_add_multi(&mut recovery, &srcs, &coeffs); + + let mut body = Vec::with_capacity(4 + slice_size); + body.extend_from_slice(&exp.to_le_bytes()); + body.extend_from_slice(&recovery); + vol_bytes.extend_from_slice(&build_packet(set_id, TYPE_RECOVERY, &body)); + } + + std::fs::write(dir.join("fixture.vol00+2.par2"), &vol_bytes).unwrap(); + dir.join("fixture.par2") + } /// Test that the basic RS encode→decode round-trip works with the D×D approach. /// 2 data blocks, 2 recovery blocks, lose both data blocks, recover. @@ -633,4 +769,62 @@ mod tests { assert_eq!(outputs[0], inputs[1], "Recovered block 1 should match"); assert_eq!(outputs[1], inputs[3], "Recovered block 3 should match"); } + + #[test] + fn test_repair_cross_file_damage_respects_main_packet_file_order() { + let dir = tempfile::tempdir().unwrap(); + let slice_size = 4096usize; + let file_a_id = [0x20; 16]; + let file_b_id = [0x10; 16]; + let file_a = vec![0x41; slice_size * 2]; + let file_b = vec![0x42; slice_size * 2]; + + let par2_path = write_par2_fixture( + dir.path(), + slice_size, + &[ + ("file_a.bin", file_a_id, file_a.clone()), + ("file_b.bin", file_b_id, file_b.clone()), + ], + &[file_a_id, file_b_id], + &[0, 1], + ); + + let file_set = crate::parse(&par2_path).unwrap(); + assert_eq!(file_set.file_order, vec![file_a_id, file_b_id]); + + { + let file_a_path = dir.path().join("file_a.bin"); + let file_b_path = dir.path().join("file_b.bin"); + let mut a = std::fs::OpenOptions::new() + .write(true) + .open(&file_a_path) + .unwrap(); + let mut b = std::fs::OpenOptions::new() + .write(true) + .open(&file_b_path) + .unwrap(); + a.seek(SeekFrom::Start(slice_size as u64)).unwrap(); + b.seek(SeekFrom::Start(slice_size as u64)).unwrap(); + a.write_all(&vec![0xDE; slice_size]).unwrap(); + b.write_all(&vec![0xAD; slice_size]).unwrap(); + } + + let pre = verify::verify(&file_set, dir.path()); + assert_eq!(pre.damaged.len(), 2); + assert_eq!(pre.blocks_needed(), 2); + assert!(pre.repair_possible); + + let repaired = repair(&file_set, dir.path()).unwrap(); + assert!(repaired.success); + + assert_eq!( + std::fs::read(dir.path().join("file_a.bin")).unwrap(), + file_a + ); + assert_eq!( + std::fs::read(dir.path().join("file_b.bin")).unwrap(), + file_b + ); + } } diff --git a/src/types.rs b/src/types.rs index 360246a..e29051f 100644 --- a/src/types.rs +++ b/src/types.rs @@ -16,6 +16,10 @@ pub struct Par2FileSet { pub recovery_set_id: Id16, /// Slice (block) size in bytes. pub slice_size: u64, + /// Source file order from the Main packet. PAR2 recovery math depends on + /// this logical input ordering, which is not guaranteed to match a sort by + /// file ID or filename. + pub file_order: Vec, /// Files described in this PAR2 set, keyed by File ID. pub files: HashMap, /// Number of recovery slices available (counted from RecoverySlice packets).