ro: implement NRO decompression and fix bugs

libraries/libstratosphere/include/stratosphere/util/util_compression.hpp

@@ -28,6 +28,7 @@ namespace ams::util {
 
     /* Decompression utilities. */
     int DecompressLZ4(void *dst, size_t dst_size, const void *src, size_t src_size);
+    int DecompressLZ4Frame(size_t* out_size, void *dst, size_t dst_size, const void *src, size_t src_size, void *src_work, size_t src_work_size, void *dst_work, size_t dst_work_size);
     size_t DecompressZstd(void *dst, size_t dst_size, const void *src, size_t src_size);
     size_t DecompressZstdWithBic(void *dst, size_t dst_size, const void *src, size_t src_size);
     bool DecompressZstdWithBic(void* workspace, size_t workspace_size, void *dst, size_t dst_size, size_t expected_dec_size, const void *src, size_t src_size);

libraries/libstratosphere/source/util/util_compression_lz4.cpp

@@ -35,5 +35,151 @@ namespace ams::util {
         /* This is just a thin wrapper around LZ4. */
         return LZ4_decompress_safe(reinterpret_cast<const char *>(src), reinterpret_cast<char *>(dst), static_cast<int>(src_size), static_cast<int>(dst_size));
     }
-
+    
+    /* TODO: This was added for ro in 23.0.0+, but it's unclear if this is even part of the util namespace. */
+    /* It's similar to the official lz4frame code, but has additional constraints and uses static work buffers. */
+    /* Evaluate if we should use lz4frame instead and implement matching frame compression. */
+    int DecompressLZ4Frame(size_t* out_size, void *dst, size_t dst_size, const void *src, size_t src_size, void *src_work, size_t src_work_size, void *dst_work, size_t dst_work_size) {
+        /* Destination buffer must be behind source buffer. */
+        if (dst > src) {
+            std::memset(dst, 0, dst_size);
+            return -1;
+        }
+        
+        const char *src_end = static_cast<const char *>(src) + src_size;
+        const char *dst_end = static_cast<const char *>(dst) + dst_size;
+        
+        if ((src_end != dst_end) || (src_work_size != 0x10000) || (dst_work_size != 0x10000)) {
+            std::memset(dst, 0, dst_size);
+            return -1;
+        }
+        
+        if (src_size < 7) {
+            std::memset(dst, 0, dst_size);
+            return -2;
+        }
+        
+        const uint32_t magic = *reinterpret_cast<const uint32_t *>(src);
+        const uint8_t flg = *reinterpret_cast<const uint8_t *>(src) + 4;
+        const uint8_t bd = *reinterpret_cast<const uint8_t *>(src) + 5;
+        const uint8_t header_checksum = *reinterpret_cast<const uint8_t *>(src) + 6;
+        
+        /* Check for LZ4F_MAGICNUMBER, validate flags and block size. */
+        if ((magic != 0x184D2204) || ((flg & 1) != 0) || (flg != 0x60) || (bd != 0x40)) {
+            std::memset(dst, 0, dst_size);
+            return -2;
+        }
+        
+        /* TODO: Implement header checksum validation. This requires xxhash, do we want to have xxhash in util? */
+        AMS_UNUSED(header_checksum);
+        
+        uint32_t src_pos = 7; /* Skip the header. */
+        uint32_t dst_pos = 0;
+        
+        /* Destination size is too small. */
+        if (dst_size < src_pos) {
+            std::memset(dst, 0, dst_size);
+            return -2;
+        }
+        
+        while (true) {
+            /* Not enough space left. */
+            if ((dst_size - src_pos) < 4) {
+                std::memset(dst, 0, dst_size);
+                return -4;
+            }
+            
+            const char* src_data_block = reinterpret_cast<const char *>(src) + src_pos;
+            char* dst_data_block = reinterpret_cast<char *>(dst) + dst_pos;
+            
+            const uint32_t block_header = *reinterpret_cast<const uint32_t *>(src_data_block);
+            
+            /* We've reached the end. */
+            if (!block_header) {
+                break;
+            }
+            
+            const uint32_t block_size = block_header & 0x7FFFFFFF;
+            
+            /* Block size is invalid. */
+            if (block_size > 0x10000) {
+                std::memset(dst, 0, dst_size);
+                return -3;
+            }
+            
+            /* Advance the data block. */
+            src_data_block += sizeof(block_header);
+            
+            /* Block size is invalid. */
+            if ((dst_end - src_data_block) < block_size) {
+                std::memset(dst, 0, dst_size);
+                return -4;
+            }
+            
+            /* Copy the data block to the source work buffer. */
+            std::memcpy(static_cast<char *>(src_work), src_data_block, block_size);
+            
+            if ((block_header & 0x80000000) != 0) {
+                /* Block size is invalid. */
+                if ((dst_end - dst_data_block) < block_size) {
+                    std::memset(dst, 0, dst_size);
+                    return -4;
+                }
+                
+                dst_pos += block_size;
+                src_pos += block_size;
+                
+                /* Destination is overrunning source. */
+                if (dst_pos > src_pos) {
+                    std::memset(dst, 0, dst_size);
+                    return -5;
+                }
+                
+                /* This is a raw block, copy it. */
+                std::memmove(dst_data_block, src_data_block, block_size); 
+            } else {
+                /* This is a compressed block, decompress it. */
+                int decompressed_size = LZ4_decompress_safe(reinterpret_cast<const char *>(src_work), reinterpret_cast<char *>(dst_work), static_cast<int>(src_work_size), static_cast<int>(dst_work_size));
+                
+                /* Decompressed size is invalid. */
+                if (decompressed_size < 1) {
+                    std::memset(dst, 0, dst_size);
+                    return -3;
+                }
+                
+                /* Decompressed size is invalid. */
+                if ((dst_end - dst_data_block) < decompressed_size) {
+                    std::memset(dst, 0, dst_size);
+                    return -4;
+                }
+                
+                dst_pos += decompressed_size;
+                src_pos += block_size;
+                
+                /* Destination is overrunning source. */
+                if (dst_pos > src_pos) {
+                    std::memset(dst, 0, dst_size);
+                    return -5;
+                }
+                
+                /* Copy the decompressed data back. */
+                std::memcpy(dst_data_block, static_cast<char *>(dst_work), decompressed_size);
+            }
+            
+            /* We've exceeded the destination's size. */
+            if (src_pos >= dst_size) {
+                std::memset(dst, 0, dst_size);
+                return -2;
+            }   
+        }
+        
+        /* We didn't consume all data. */
+        if (src_pos != src_size) {
+            std::memset(dst, 0, dst_size);
+            return -2;
+        }
+        
+        *out_size = dst_pos;
+        return 0;
+    }
 }

stratosphere/ro/source/impl/ro_service_impl.cpp

@@ -94,6 +94,8 @@ namespace ams::ro::impl {
                 os::NativeHandle m_process_handle = os::InvalidNativeHandle;
                 os::ProcessId m_process_id = os::InvalidProcessId;
                 bool m_in_use{};
+                u8  m_decompression_src_work_buffer[0x10000]{};
+                u8  m_decompression_dst_work_buffer[0x10000]{};
             public:
                 constexpr ProcessContext() = default;
 
@@ -108,6 +110,9 @@ namespace ams::ro::impl {
                     m_process_handle = process_handle;
                     m_process_id     = process_id;
                     m_in_use         = true;
+                    
+                    std::memset(m_decompression_src_work_buffer, 0, sizeof(m_decompression_src_work_buffer));
+                    std::memset(m_decompression_dst_work_buffer, 0, sizeof(m_decompression_dst_work_buffer));
                 }
 
                 void Finalize() {
@@ -259,7 +264,7 @@ namespace ams::ro::impl {
                     R_THROW(ro::ResultNotAuthorized());
                 }
                 
-                Result ValidateNro(NroHeader **out_header, AdditionalNroHeaderInfo **out_additional_header_info, void *out_additional_header_hash, u8 *mapped_memory, u64 expected_nro_size, u64 expected_bss_size) {
+                Result ValidateNro(NroHeader **out_header, AdditionalNroHeaderInfo *out_additional_header_info, void *out_additional_header_hash, u8 *mapped_memory, u64 expected_nro_size, u64 expected_bss_size) {
                     /* Validate header. */
                     NroHeader *header = reinterpret_cast<NroHeader *>(mapped_memory);
                     R_UNLESS(header->IsMagicValid(), ro::ResultInvalidNro());
@@ -348,11 +353,11 @@ namespace ams::ro::impl {
                         R_UNLESS(util::AlignUp(compressed_size, os::MemoryPageSize) == expected_nro_size, ro::ResultInvalidNro());
                         
                         /* Copy the additional header info to output. This is AdditionalNroHeader minus the hash. */
-                        std::memcpy(out_additional_header_info, additional_header, sizeof(*additional_header) - hash_size);
+                        std::memcpy(reinterpret_cast<void*>(out_additional_header_info), additional_header, sizeof(AdditionalNroHeader) - hash_size);
                     } else {
                         /* Explicitly clear additional header hash and info. */
                         std::memset(out_additional_header_hash, 0, crypto::Sha256Generator::HashSize);
-                        std::memset(out_additional_header_info, 0, sizeof(AdditionalNroHeaderInfo));
+                        std::memset(reinterpret_cast<void*>(out_additional_header_info), 0, sizeof(AdditionalNroHeaderInfo));
                     }
                     
                     /* Copy the header to output. */
@@ -361,9 +366,48 @@ namespace ams::ro::impl {
                 }
                 
                 Result DecompressNro(u8 *mapped_nro, u64 expected_nro_size, const NroHeader *nro_header, u32 compressed_size) {
-                    /* TODO */
+                    /* Read sizes from header. */
-                    AMS_UNUSED(mapped_nro, expected_nro_size, nro_header, compressed_size);
+                    const u64 rw_ofs = nro_header->GetRwOffset();
-                    R_SUCCEED();
+                    const u64 rw_size = nro_header->GetRwSize();
+                    const u64 bss_size = nro_header->GetBssSize();
+                    
+                    /* Divide the mapped NRO into regions devised for the decompression. */
+                    u8 *nro_aligned_start = static_cast<u8 *>(mapped_nro + 0x1000);
+                    const u8 *nro_expected_end = static_cast<const u8 *>(mapped_nro + expected_nro_size);
+                    const u64 nro_src_buffer_size = compressed_size - 0x1000;
+                    u8 *nro_src_buffer = static_cast<u8 *>(mapped_nro + expected_nro_size - nro_src_buffer_size);
+                    const u64 nro_dst_buffer_size = expected_nro_size - 0xC00;
+                    u8 *nro_dst_buffer = static_cast<u8 *>(mapped_nro + 0xC00);
+                    
+                    /* Copy to source buffer. */
+                    std::memmove(nro_src_buffer, nro_aligned_start, nro_src_buffer_size);
+                    
+                    /* Decompress. */
+                    u64 decompressed_size = 0;
+                    R_UNLESS(util::DecompressLZ4Frame(std::addressof(decompressed_size), nro_dst_buffer, nro_dst_buffer_size, nro_src_buffer, nro_src_buffer_size, m_decompression_src_work_buffer, sizeof(m_decompression_src_work_buffer), m_decompression_dst_work_buffer, sizeof(m_decompression_dst_work_buffer)), ro::ResultInvalidNro());
+                    
+                    /* Find the BSS. */
+                    u8 *nro_expected_bss_start = static_cast<u8 *>(mapped_nro + rw_ofs + rw_size);
+                    const u8 *nro_expected_bss_end = static_cast<const u8 *>(nro_expected_bss_start + bss_size);
+                    const u8 *nro_bss_start = static_cast<const u8 *>(nro_expected_bss_start - 0x400);
+                    const u8 *nro_decompressed_end = static_cast<const u8 *>(nro_dst_buffer + decompressed_size);
+                    
+                    /* The decompressed data must end at the expected BSS start. */
+                    if (nro_decompressed_end == nro_bss_start) {
+                        /* Copy the decompressed data back to the mapped NRO memory. */
+                        std::memmove(nro_aligned_start, nro_dst_buffer, decompressed_size);
+                        std::memset(nro_dst_buffer, 0, 0x400);
+                        
+                        /* Check the BSS ends where it's supposed, initialize it and then succeed. */
+                        if (nro_expected_bss_end == nro_expected_end) {
+                            std::memset(nro_expected_bss_start, 0, bss_size);
+                            R_SUCCEED();
+                        }
+                    }
+                    
+                    /* Clear the decompressed data buffer in case of error. */
+                    std::memset(nro_dst_buffer, 0, nro_dst_buffer_size);
+                    R_THROW(ro::ResultInvalidNro());
                 }
                 
                 Result CheckAdditionalHeaderHash(const u8 *mapped_nro, const void *additional_nro_header_hash, size_t additional_nro_header_hash_size, u64 rw_ofs, u64 rw_size) {
@@ -391,13 +435,13 @@ namespace ams::ro::impl {
                     
                     /* ValidateNro */
                     NroHeader *nro_header = nullptr;
-                    AdditionalNroHeaderInfo *additional_nro_header_info = nullptr;
+                    AdditionalNroHeaderInfo additional_nro_header_info;
                     u8 additional_nro_header_hash[crypto::Sha256Generator::HashSize];
                     R_TRY(this->ValidateNro(std::addressof(nro_header), std::addressof(additional_nro_header_info), std::addressof(additional_nro_header_hash), static_cast<u8 *>(mapped_memory), expected_nro_size, expected_bss_size));
                     
                     /* DecompressNro */
                     if (nro_header->IsCompress()) {
-                        R_TRY(this->DecompressNro(static_cast<u8 *>(mapped_memory), expected_nro_size, nro_header, additional_nro_header_info->compressed_size));
+                        R_TRY(this->DecompressNro(static_cast<u8 *>(mapped_memory), expected_nro_size, nro_header, additional_nro_header_info.compressed_size));
                         
                         /* CheckAdditionalHeaderHash */
                         if (nro_header->GetAdditionalHeaderOffset() != 0) {