diff --git a/AsarSharp/AsarCreator.cs b/AsarSharp/AsarCreator.cs index a4bea28..9b1276f 100644 --- a/AsarSharp/AsarCreator.cs +++ b/AsarSharp/AsarCreator.cs @@ -36,7 +36,6 @@ namespace AsarSharp CreatePackageFromFiles(); } - public void CreatePackageFromFiles() { var filesystem = new Filesystem(_folderPath); @@ -50,7 +49,6 @@ namespace AsarSharp InsertsDone(filesystem, files); } - private void HandleFile(Filesystem filesystem, string filename, List files) { if (!_metadata.TryGetValue(filename, out var file)) @@ -69,14 +67,9 @@ namespace AsarSharp string parentDir = Path.GetDirectoryName(filename) ?? string.Empty; string relParent = Extensions.GetRelativePath(_folderPath, parentDir); bool shouldUnpack = ShouldUnpackPath(relParent); + long fileSize = file.Stat is FileInfo fi ? fi.Length : 0; + var placeholder = IntegrityHelper.CreatePlaceholder(fileSize); files.Add(new Disk.BasicFileInfo { Filename = filename, Unpack = shouldUnpack }); - - // Build a placeholder integrity record up front. Real - // SHA-256 hashes are filled in by Disk.WriteFileSystem - // during the streamed write — eliminates the second pass - // over each file (open → hash → close → open → copy → close). - long size = (file.Stat is FileInfo fi) ? fi.Length : 0; - var placeholder = IntegrityHelper.CreatePlaceholder(size); filesystem.InsertFile(filename, shouldUnpack, file, placeholder); break; case FileType.Link: diff --git a/AsarSharp/AsarFileSystem/Disk.cs b/AsarSharp/AsarFileSystem/Disk.cs index 16e3f18..d83e11a 100644 --- a/AsarSharp/AsarFileSystem/Disk.cs +++ b/AsarSharp/AsarFileSystem/Disk.cs @@ -1,6 +1,8 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; +using AsarSharp.Integrity; using AsarSharp.PickleTools; using AsarSharp.Utils; using Newtonsoft.Json; @@ -10,7 +12,8 @@ namespace AsarSharp.AsarFileSystem public static class Disk { private const int StreamBufferSize = 1024 * 1024; - private static Dictionary _filesystemCache = new Dictionary(); + private static readonly ConcurrentDictionary _filesystemCache = + new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); public class ArchiveHeader { @@ -30,36 +33,29 @@ namespace AsarSharp.AsarFileSystem public string Filename { get; set; } public bool Unpack { get; set; } } - - + #region Reading - + public static ArchiveHeader ReadArchiveHeaderSync(string archivePath) { - using (var fs = new FileStream(archivePath, FileMode.Open, FileAccess.Read, FileShare.Read, StreamBufferSize, FileOptions.SequentialScan)) - { - // read the size of the header (8 bytes) + using (var fs = new FileStream(archivePath, FileMode.Open, FileAccess.Read, FileShare.Read, + 65536, FileOptions.SequentialScan)) + { byte[] sizeBuf = new byte[8]; if (fs.Read(sizeBuf, 0, 8) != 8) - { throw new Exception("Unable to read header size"); - } - + var sizePickle = Pickle.CreateFromBuffer(sizeBuf); var size = sizePickle.CreateIterator().ReadUInt32(); - - // Read the header of the specified size + var headerBuf = new byte[size]; - if(fs.Read(headerBuf, 0, (int)size) != size) - { + if (fs.Read(headerBuf, 0, (int)size) != size) throw new Exception("Unable to read header"); - } - + var headerPickle = Pickle.CreateFromBuffer(headerBuf); var header = headerPickle.CreateIterator().ReadString(); - var headerObj = JsonConvert.DeserializeObject(header); - + return new ArchiveHeader { Header = headerObj, @@ -68,82 +64,62 @@ namespace AsarSharp.AsarFileSystem }; } } + public static Filesystem ReadFilesystemSync(string archivePath) { - if (!_filesystemCache.ContainsKey(archivePath) || _filesystemCache[archivePath] == null) + return _filesystemCache.GetOrAdd(archivePath, key => { - ArchiveHeader header = ReadArchiveHeaderSync(archivePath); - Filesystem filesystem = new Filesystem(archivePath); + var header = ReadArchiveHeaderSync(key); + var filesystem = new Filesystem(key); filesystem.SetHeader(header.Header, header.HeaderSize); - _filesystemCache[archivePath] = filesystem; - } - - return _filesystemCache[archivePath]; + return filesystem; + }); } - + public static byte[] ReadFileSync(Filesystem filesystem, string filename, FilesystemEntry info) { if (!info.IsFile || !info.Size.HasValue) - { throw new ArgumentException("Entry is not a file", nameof(info)); - } long size = info.Size.Value; byte[] buffer = new byte[size]; - - if (size <= 0) - { - return buffer; - } - + + if (size <= 0) return buffer; + if (info.Unpacked == true) { - // It's an unpacked file, read it directly string filePath = Path.Combine($"{filesystem.GetRootPath()}.unpacked", filename); return File.ReadAllBytes(filePath); } - // Read from the ASAR archive - using (var fs = new FileStream(filesystem.GetRootPath(), FileMode.Open, FileAccess.Read, FileShare.Read, StreamBufferSize, FileOptions.SequentialScan)) + using (var fs = new FileStream(filesystem.GetRootPath(), FileMode.Open, FileAccess.Read, + FileShare.Read, 65536, FileOptions.RandomAccess)) { - // Important: the offset must take into account the size of the Pickle header (8 bytes) - // and the size of the header itself long offset = 8 + filesystem.GetHeaderSize() + long.Parse(info.Offset); fs.Position = offset; - - // Read the whole file at once int bytesRead = fs.Read(buffer, 0, (int)size); if (bytesRead != size) - { throw new Exception($"Failed to read entire file, got {bytesRead} bytes instead of {size}"); - } } - - + return buffer; } - + #endregion - + public static bool UncacheFilesystem(string archivePath) { - if (_filesystemCache.ContainsKey(archivePath)) - { - _filesystemCache.Remove(archivePath); - return true; - } - - return false; + return _filesystemCache.TryRemove(archivePath, out _); } public static void UncacheAll() { _filesystemCache.Clear(); } - + public static void CopyFile(string dest, string rootPath, string filename) { - if(dest == null || rootPath == null || filename == null) + if (dest == null || rootPath == null || filename == null) throw new ArgumentNullException(); string normalizedDestRoot = Path.GetFullPath(dest) @@ -152,63 +128,97 @@ namespace AsarSharp.AsarFileSystem .TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); if (string.Equals(normalizedDestRoot, normalizedRootPath, StringComparison.OrdinalIgnoreCase)) - { return; - } - + string sourcePath = Path.GetFullPath(Path.Combine(rootPath, filename)); string destPath = Path.GetFullPath(Path.Combine(dest, filename)); if (string.Equals(sourcePath, destPath, StringComparison.OrdinalIgnoreCase)) - { return; - } Directory.CreateDirectory(Path.GetDirectoryName(destPath) ?? throw new InvalidOperationException()); - using (var sourceStream = new FileStream(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read, StreamBufferSize, FileOptions.SequentialScan)) - using (var destinationStream = new FileStream(destPath, FileMode.Create, FileAccess.Write, FileShare.None, StreamBufferSize, FileOptions.SequentialScan)) + using (var src = new FileStream(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read, StreamBufferSize, FileOptions.SequentialScan)) + using (var dst = new FileStream(destPath, FileMode.Create, FileAccess.Write, FileShare.None, StreamBufferSize, FileOptions.SequentialScan)) { - sourceStream.CopyTo(destinationStream, StreamBufferSize); + src.CopyTo(dst, StreamBufferSize); } } - public static void WriteFileSystem(string dest, Filesystem fileSystem, - FilesystemFilesAndLinks lists, - Dictionary metadata) + FilesystemFilesAndLinks lists, Dictionary metadata) { - var fsHeader = fileSystem.GetHeader(); - var headerPickle = Pickle.CreateEmpty(); - var serializerSettings = new JsonSerializerSettings() - { NullValueHandling = NullValueHandling.Ignore, DefaultValueHandling = DefaultValueHandling.Ignore } ; + var serializerSettings = new JsonSerializerSettings + { + NullValueHandling = NullValueHandling.Ignore, + DefaultValueHandling = DefaultValueHandling.Ignore + }; - var headerJson = JsonConvert.SerializeObject(fsHeader,serializerSettings); + // --- Phase 1: write placeholder header --- + string headerJson = JsonConvert.SerializeObject(fileSystem.GetHeader(), serializerSettings); + var headerPickle = Pickle.CreateEmpty(); headerPickle.WriteString(headerJson); - var headerBuf = headerPickle.ToBuffer(); - + var sizePickle = Pickle.CreateEmpty(); - sizePickle.WriteUInt32((uint)headerBuf.Length); - var sizeBuf = sizePickle.ToBuffer(); - + sizePickle.WriteUInt32((uint)headerPickle.GetTotalSize()); + int sizePickleSize = sizePickle.GetTotalSize(); + + var buf = new byte[StreamBufferSize]; + var blockBuf = new byte[4 * 1024 * 1024]; // shared across all files — avoids 4MB alloc per file + using (var fs = new FileStream(dest, FileMode.Create, FileAccess.Write, FileShare.None, StreamBufferSize, FileOptions.SequentialScan)) { - fs.Write(sizeBuf, 0, sizeBuf.Length); - fs.Write(headerBuf, 0, headerBuf.Length); - + sizePickle.WriteTo(fs); + headerPickle.WriteTo(fs); + + // --- Phase 2: stream files, hash in one pass, patch nodes in-memory --- foreach (var file in lists.Files) { if (file.Unpack) { - var filename = Extensions.GetRelativePath(fileSystem.GetRootPath(), file.Filename); - CopyFile($"{dest}.unpacked", fileSystem.GetRootPath(), filename); + var relName = Extensions.GetRelativePath(fileSystem.GetRootPath(), file.Filename); + CopyFile($"{dest}.unpacked", fileSystem.GetRootPath(), relName); + CopyAndHash(file.Filename, null, buf, blockBuf, fileSystem); continue; } - using (var transformedFileStream = new FileStream(file.Filename, FileMode.Open, FileAccess.Read, FileShare.Read, StreamBufferSize, FileOptions.SequentialScan)) - { - transformedFileStream.CopyTo(fs, StreamBufferSize); - } + + CopyAndHash(file.Filename, fs, buf, blockBuf, fileSystem); } + + // --- Phase 3: re-serialize header with real hashes, seek back, overwrite --- + string patchedJson = JsonConvert.SerializeObject(fileSystem.GetHeader(), serializerSettings); + var patchedPickle = Pickle.CreateEmpty(); + patchedPickle.WriteString(patchedJson); + + var patchedSizePickle = Pickle.CreateEmpty(); + patchedSizePickle.WriteUInt32((uint)patchedPickle.GetTotalSize()); + + fs.Position = 0; + patchedSizePickle.WriteTo(fs); + patchedPickle.WriteTo(fs); + } + } + + private static void CopyAndHash(string srcPath, Stream dest, byte[] buf, byte[] blockBuf, Filesystem fs) + { + string relPath = Extensions.GetRelativePath(fs.GetRootPath(), srcPath); + var node = fs.GetNode(relPath, followLinks: false); + + long fileSize = node?.Size ?? 0; + int estimatedBlocks = fileSize > 0 ? (int)((fileSize + 4 * 1024 * 1024 - 1) / (4 * 1024 * 1024)) : 0; + + using (var hasher = new IntegrityHelper.StreamingHasher(estimatedBlocks, blockBuf)) + using (var src = new FileStream(srcPath, FileMode.Open, FileAccess.Read, FileShare.Read, StreamBufferSize, FileOptions.SequentialScan)) + { + int read; + while ((read = src.Read(buf, 0, buf.Length)) > 0) + { + hasher.Append(buf, 0, read); + dest?.Write(buf, 0, read); + } + + if (node != null) + node.Integrity = hasher.Finalise(); } } } -} \ No newline at end of file +} diff --git a/AsarSharp/AsarFileSystem/FileSystem.cs b/AsarSharp/AsarFileSystem/FileSystem.cs index e8cc673..e577c4e 100644 --- a/AsarSharp/AsarFileSystem/FileSystem.cs +++ b/AsarSharp/AsarFileSystem/FileSystem.cs @@ -13,7 +13,7 @@ namespace AsarSharp.AsarFileSystem private int _headerSize; private long _offset; - private const uint UINT32_MAX = 0xFFFFFFFF; // 2^32 - 1 + private const uint UINT32_MAX = 0xFFFFFFFF; public Filesystem(string src) { @@ -23,20 +23,9 @@ namespace AsarSharp.AsarFileSystem _offset = 0; } - public string GetRootPath() - { - return _src; - } - - public FilesystemEntry GetHeader() - { - return _header; - } - - public int GetHeaderSize() - { - return _headerSize; - } + public string GetRootPath() => _src; + public FilesystemEntry GetHeader() => _header; + public int GetHeaderSize() => _headerSize; public void SetHeader(FilesystemEntry header, int headerSize) { @@ -47,82 +36,94 @@ namespace AsarSharp.AsarFileSystem public FilesystemEntry SearchNodeFromDirectory(string p) { FilesystemEntry json = _header; - - // Normalize path delimiters to system delimiters - p = p.Replace('/', Path.DirectorySeparatorChar).Replace('\\', Path.DirectorySeparatorChar); - - string[] dirs = p.Split(Path.DirectorySeparatorChar); - - foreach (string dir in dirs) + + int len = p.Length; + int start = 0; + + // skip leading separators + while (start < len && (p[start] == '/' || p[start] == '\\')) start++; + + while (start < len) { - if (dir == "." || string.IsNullOrEmpty(dir)) continue; - - if (json.IsDirectory) + // find next separator + int end = start; + while (end < len && p[end] != '/' && p[end] != '\\') end++; + + int segLen = end - start; + if (segLen == 0 || (segLen == 1 && p[start] == '.')) { - if (!json.Files.ContainsKey(dir)) - { - json.Files[dir] = new FilesystemEntry { Files = new Dictionary(StringComparer.Ordinal) }; - } - json = json.Files[dir]; + start = end + 1; + continue; } - else - { + + string seg = p.Substring(start, segLen); + + if (!json.IsDirectory) throw new Exception($"Unexpected directory state while traversing: {p}"); + + if (!json.Files.TryGetValue(seg, out var child)) + { + child = new FilesystemEntry { Files = new Dictionary(StringComparer.Ordinal) }; + json.Files[seg] = child; } + json = child; + start = end + 1; } - + return json; } - + + public (FilesystemEntry parent, string name) SearchNodeFromPathWithParent(string p) + { + string rel = Extensions.GetRelativePath(_src, p); + if (string.IsNullOrEmpty(rel)) + return (_header, string.Empty); + + string name = Path.GetFileName(rel); + string dir = Extensions.GetDirectoryName(rel); + var parent = SearchNodeFromDirectory(dir); + + if (parent.Files == null) + parent.Files = new Dictionary(StringComparer.Ordinal); + + if (!parent.Files.ContainsKey(name)) + parent.Files[name] = new FilesystemEntry(); + + return (parent, name); + } + public List ListFiles(bool isPack = false) { var files = new List(); - FillFilesFromMetadata("/", _header); return files; void FillFilesFromMetadata(string basePath, FilesystemEntry metadata) { - if (!metadata.IsDirectory) - { - return; - } - + if (!metadata.IsDirectory) return; foreach (var entry in metadata.Files) { - string childPath = entry.Key; - FilesystemEntry childMetadata = entry.Value; - string fullPath = Path.Combine(basePath, childPath).Replace('\\', '/'); - - string packState = - childMetadata.Unpacked == true ? "unpack" : "pack "; - + string fullPath = Path.Combine(basePath, entry.Key).Replace('\\', '/'); + string packState = entry.Value.Unpacked == true ? "unpack" : "pack "; files.Add(isPack ? $"{packState} : {fullPath}" : fullPath); - FillFilesFromMetadata(fullPath, childMetadata); + FillFilesFromMetadata(fullPath, entry.Value); } } } public FilesystemEntry GetNode(string p, bool followLinks = true) { - // Normalize path delimiters p = p.Replace('/', Path.DirectorySeparatorChar).Replace('\\', Path.DirectorySeparatorChar); - FilesystemEntry node = SearchNodeFromDirectory(Extensions.GetDirectoryName(p)); string name = Path.GetFileName(p); - - // Process symbolic links + if (node.IsLink && followLinks) - { return GetNode(Path.Combine(node.Link, name)); - } - + if (!string.IsNullOrEmpty(name)) { if (node.IsDirectory && node.Files.TryGetValue(name, out var entry)) - { return entry; - } return null; } @@ -132,103 +133,62 @@ namespace AsarSharp.AsarFileSystem public FilesystemEntry GetFile(string p, bool followLinks = true) { FilesystemEntry info = GetNode(p, followLinks); - - if (info == null) - { - throw new Exception($"\"{p}\" was not found in this archive"); - } - - // If followLinks=false, do not allow symbolic links (TODO) - if (info.IsLink && followLinks) - { - return GetFile(info.Link, followLinks); - } - + if (info == null) throw new Exception($"\"{p}\" was not found in this archive"); + if (info.IsLink && followLinks) return GetFile(info.Link, followLinks); return info; } - - public static string ReadLink(string path) - { - throw new NotImplementedException(); - // TODO , NOT IMPLEMENTED - } - - - + + public static string ReadLink(string path) => throw new NotImplementedException(); + #region Writing - + public FilesystemEntry SearchNodeFromPath(string p) { - p = Extensions.GetRelativePath(_src, p); - - if (string.IsNullOrEmpty(p)) - { - return _header; - } - - var name = Path.GetFileName(p); - var node = SearchNodeFromDirectory(Extensions.GetDirectoryName(p)); - - if (node.Files == null) - { - node.Files = new Dictionary(); - } - - if (!node.Files.ContainsKey(name)) - { - node.Files[name] = new FilesystemEntry(); - } - - return node.Files[name]; + var (parent, name) = SearchNodeFromPathWithParent(p); + if (string.IsNullOrEmpty(name)) return _header; + return parent.Files[name]; } - + public void InsertDirectory(string p, bool unpack) { FilesystemEntry node = SearchNodeFromPath(p); - node.Files = node.Files ?? new Dictionary(); + node.Files = node.Files ?? new Dictionary(StringComparer.Ordinal); node.Unpacked = unpack; } - - public void InsertFile(string path, bool shouldUnpack, CrawledFileType file) + + public void InsertFile(string path, bool shouldUnpack, CrawledFileType file, + IntegrityHelper.FileIntegrity precomputedIntegrity = null) { - var dirName = Path.GetDirectoryName(path); - var dirNode = SearchNodeFromPath(dirName); + var (dirNode, _) = SearchNodeFromPathWithParent(Path.GetDirectoryName(path) ?? path); var node = SearchNodeFromPath(path); - long size = 0; - if (file.Stat is FileInfo fileInfo) - { - size = fileInfo.Length; - } + long size; + if (file.Stat is FileInfo fi) + size = fi.Length; else - { throw new Exception($"{path}: stat is not a file"); - } - + if (shouldUnpack || dirNode.Unpacked == true) { node.Size = size; node.Unpacked = true; - node.Integrity = IntegrityHelper.GetFileIntegrity(path); + node.Integrity = precomputedIntegrity ?? IntegrityHelper.GetFileIntegrity(path); return; } - // Check that the file size does not exceed UINT32_MAX if (size > UINT32_MAX) - { throw new Exception($"{path}: file size cannot be larger than 4.2GB"); - } node.Size = size; node.Offset = _offset.ToString(); - node.Integrity = IntegrityHelper.GetFileIntegrity(path); + node.Integrity = precomputedIntegrity ?? IntegrityHelper.GetFileIntegrity(path); + if (!Extensions.IsWindowsPlatform() && (file.Stat.Attributes & FileAttributes.Hidden) != 0) - { node.Executable = true; - } + _offset += size; } - + #endregion } -} \ No newline at end of file +} diff --git a/AsarSharp/AsarFileSystem/FileSystemCrawler.cs b/AsarSharp/AsarFileSystem/FileSystemCrawler.cs index 6460ec8..a4ee35e 100644 --- a/AsarSharp/AsarFileSystem/FileSystemCrawler.cs +++ b/AsarSharp/AsarFileSystem/FileSystemCrawler.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; using AsarSharp.Utils; namespace AsarSharp.AsarFileSystem @@ -25,7 +24,7 @@ namespace AsarSharp.AsarFileSystem Directory, Link } - + public static class FileSystemCrawler { public static CrawledFileType DetermineFileType(string filename) @@ -46,124 +45,87 @@ namespace AsarSharp.AsarFileSystem ? (FileSystemInfo)new DirectoryInfo(filename) : new FileInfo(filename); - if (isLink) - { - return new CrawledFileType { Type = FileType.Link, Stat = info }; - } - - if (isDirectory) - { - return new CrawledFileType { Type = FileType.Directory, Stat = info }; - } - + if (isLink) return new CrawledFileType { Type = FileType.Link, Stat = info }; + if (isDirectory) return new CrawledFileType { Type = FileType.Directory, Stat = info }; return new CrawledFileType { Type = FileType.File, Stat = info }; } public static (List filenames, Dictionary metadata) CrawlFileSystem(string dir) { var metadata = new Dictionary(); - var crawled = CrawlIterative(dir); - var results = crawled.Select(filename => new { filename, type = DetermineFileType(filename) }).ToList(); - - var links = new List(); var filenames = new List(); + var links = new List(); - foreach (var result in results.Where(result => result.type != null)) + foreach (var fullPath in CrawlIterative(dir)) { - metadata[result.filename] = result.type; - if (result.type.Type == FileType.Link) - { - links.Add(result.filename); - } - filenames.Add(result.filename); + var type = DetermineFileType(fullPath); + if (type == null) continue; + metadata[fullPath] = type; + if (type.Type == FileType.Link) links.Add(fullPath); + filenames.Add(fullPath); } - if (links.Count == 0) - { - return (filenames, metadata); - } - - var filteredFilenames = new List(filenames.Count); + if (links.Count == 0) return (filenames, metadata); + var filtered = new List(filenames.Count); foreach (var filename in filenames) { - var exactLinkIndex = links.FindIndex(link => filename == link); - var isValid = true; + bool isValid = true; + string fileDir = Path.GetDirectoryName(filename) ?? string.Empty; - for (var i = 0; i < links.Count; i++) + foreach (var link in links) { - if (i == exactLinkIndex) - { - continue; - } + if (string.Equals(filename, link, StringComparison.OrdinalIgnoreCase)) continue; - var link = links[i]; - var isFileWithinSymlinkDir = filename.StartsWith(link, StringComparison.OrdinalIgnoreCase); - var relativePath = Extensions.GetRelativePath(link, Path.GetDirectoryName(filename) ?? string.Empty); - - if (isFileWithinSymlinkDir && !relativePath.StartsWith("..", StringComparison.Ordinal)) + if (filename.StartsWith(link, StringComparison.OrdinalIgnoreCase)) { - isValid = false; - break; + string rel = Extensions.GetRelativePath(link, fileDir); + if (!rel.StartsWith("..", StringComparison.Ordinal)) + { + isValid = false; + break; + } } } - if (isValid) - { - filteredFilenames.Add(filename); - } + if (isValid) filtered.Add(filename); } - return (filteredFilenames, metadata); + return (filtered, metadata); } - // (File order is not important!!!) public static List CrawlIterative(string dir) { var result = new List(); - var stack = new Stack(); + var stack = new Stack(); - string basePath = Extensions.GetBasePath(dir); + if (!Directory.Exists(basePath)) return result; - if (!Directory.Exists(basePath)) - return result; - - // Add only the base directory to the stack, but not to the result - stack.Push(basePath); + stack.Push(new DirectoryInfo(basePath)); while (stack.Count > 0) { - string currentDir = stack.Pop(); - + var current = stack.Pop(); + FileSystemInfo[] entries; try { - // Add all files from the current directory - result.AddRange(Directory.GetFiles(currentDir, "*", SearchOption.TopDirectoryOnly)); - - // Add subdirectories to the results and to the stack - foreach (var directory in Directory.GetDirectories(currentDir, "*", - SearchOption.TopDirectoryOnly)) - { - // Add subdirectories to the result - if (directory != basePath) // Do not add a base directory - { - result.Add(directory); - } - - // Add to the stack for processing - stack.Push(directory); - } + entries = current.GetFileSystemInfos(); } catch (UnauthorizedAccessException) { - // Skip directories to which there is no access continue; } + + foreach (var entry in entries) + { + result.Add(entry.FullName); + if (entry is DirectoryInfo subDir) + stack.Push(subDir); + } } - return result; } } -} \ No newline at end of file +} diff --git a/AsarSharp/Integrity/IntegrityHelper.cs b/AsarSharp/Integrity/IntegrityHelper.cs index 0085c35..f6c5302 100644 --- a/AsarSharp/Integrity/IntegrityHelper.cs +++ b/AsarSharp/Integrity/IntegrityHelper.cs @@ -9,77 +9,148 @@ namespace AsarSharp.Integrity public static class IntegrityHelper { private const string ALGORITHM = "SHA256"; - // 4MB default block size private const int BLOCK_SIZE = 4 * 1024 * 1024; + public const string PLACEHOLDER_HASH = "0000000000000000000000000000000000000000000000000000000000000000"; private static readonly char[] HexDigits = "0123456789abcdef".ToCharArray(); public class FileIntegrity { [JsonProperty("algorithm")] public string Algorithm { get; set; } - + [JsonProperty("hash")] public string Hash { get; set; } - + [JsonProperty("blockSize")] public int BlockSize { get; set; } - + [JsonProperty("blocks")] public List Blocks { get; set; } } - public static FileIntegrity GetFileIntegrity(string path) + public static FileIntegrity CreatePlaceholder(long fileSize) { - using (var fileStream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, BLOCK_SIZE, FileOptions.SequentialScan)) - using(var fileHash = SHA256.Create()) + int blockCount = fileSize > 0 ? (int)((fileSize + BLOCK_SIZE - 1) / BLOCK_SIZE) : 0; + var blocks = new List(blockCount); + for (int i = 0; i < blockCount; i++) + blocks.Add(PLACEHOLDER_HASH); + + return new FileIntegrity + { + Algorithm = ALGORITHM, + Hash = PLACEHOLDER_HASH, + BlockSize = BLOCK_SIZE, + Blocks = blocks, + }; + } + + public static FileIntegrity GetFileIntegrity(string path, byte[] reusableBuffer = null) + { + bool ownBuffer = reusableBuffer == null; + if (ownBuffer) reusableBuffer = new byte[BLOCK_SIZE]; + + using (var fileStream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, + 65536, FileOptions.SequentialScan)) + using (var fileHash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256)) using (var blockHash = SHA256.Create()) { int estimatedBlockCount = fileStream.Length > 0 ? (int)((fileStream.Length + BLOCK_SIZE - 1) / BLOCK_SIZE) : 0; var blockHashes = new List(estimatedBlockCount); - var buffer = new byte[BLOCK_SIZE]; int bytesRead; - while ((bytesRead = fileStream.Read(buffer, 0, buffer.Length)) > 0) + while ((bytesRead = fileStream.Read(reusableBuffer, 0, reusableBuffer.Length)) > 0) { - blockHashes.Add(HashBlock(blockHash, buffer, bytesRead)); - fileHash.TransformBlock(buffer, 0, bytesRead, null, 0); + blockHashes.Add(ToLowerHex(blockHash.ComputeHash(reusableBuffer, 0, bytesRead))); + fileHash.AppendData(reusableBuffer, 0, bytesRead); } - fileHash.TransformFinalBlock(Array.Empty(), 0, 0); - return new FileIntegrity { Algorithm = ALGORITHM, - Hash = ToLowerHex(fileHash.Hash), + Hash = ToLowerHex(fileHash.GetHashAndReset()), BlockSize = BLOCK_SIZE, Blocks = blockHashes, }; } } - private static string HashBlock(HashAlgorithm hashAlgorithm, byte[] buffer, int bytesRead) + public sealed class StreamingHasher : IDisposable { - return ToLowerHex(hashAlgorithm.ComputeHash(buffer, 0, bytesRead)); + private readonly IncrementalHash _fileHash; + private readonly SHA256 _blockHash; + private readonly byte[] _blockBuf; + private int _blockFill; + private readonly List _blockHashes; + + public StreamingHasher(int estimatedBlocks = 0, byte[] sharedBlockBuffer = null) + { + _fileHash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + _blockHash = SHA256.Create(); + _blockBuf = sharedBlockBuffer ?? new byte[BLOCK_SIZE]; + _blockFill = 0; + _blockHashes = new List(estimatedBlocks); + } + + public void Append(byte[] data, int offset, int count) + { + _fileHash.AppendData(data, offset, count); + + int remaining = count; + int src = offset; + while (remaining > 0) + { + int space = BLOCK_SIZE - _blockFill; + int copy = Math.Min(space, remaining); + Buffer.BlockCopy(data, src, _blockBuf, _blockFill, copy); + _blockFill += copy; + src += copy; + remaining -= copy; + + if (_blockFill == BLOCK_SIZE) + { + _blockHashes.Add(ToLowerHex(_blockHash.ComputeHash(_blockBuf, 0, BLOCK_SIZE))); + _blockFill = 0; + } + } + } + + public FileIntegrity Finalise() + { + if (_blockFill > 0) + { + _blockHashes.Add(ToLowerHex(_blockHash.ComputeHash(_blockBuf, 0, _blockFill))); + _blockFill = 0; + } + + return new FileIntegrity + { + Algorithm = ALGORITHM, + Hash = ToLowerHex(_fileHash.GetHashAndReset()), + BlockSize = BLOCK_SIZE, + Blocks = _blockHashes, + }; + } + + public void Dispose() + { + _fileHash.Dispose(); + _blockHash.Dispose(); + } } - private static string ToLowerHex(byte[] bytes) + public static string ToLowerHex(byte[] bytes) { - if (bytes == null || bytes.Length == 0) - { - return string.Empty; - } - + if (bytes == null || bytes.Length == 0) return string.Empty; var chars = new char[bytes.Length * 2]; - for (int index = 0; index < bytes.Length; index++) + for (int i = 0; i < bytes.Length; i++) { - byte value = bytes[index]; - chars[index * 2] = HexDigits[value >> 4]; - chars[index * 2 + 1] = HexDigits[value & 0x0F]; + byte v = bytes[i]; + chars[i * 2] = HexDigits[v >> 4]; + chars[i * 2 + 1] = HexDigits[v & 0x0F]; } - return new string(chars); } } -} \ No newline at end of file +} diff --git a/build.ps1 b/build.ps1 index de2e31e..f4ee2b4 100644 --- a/build.ps1 +++ b/build.ps1 @@ -23,6 +23,26 @@ function Resolve-CommandPath { return $command.Source } +function Resolve-NuGetPath { + $nugetCommand = Get-Command 'nuget.exe' -ErrorAction SilentlyContinue + if (-not $nugetCommand) { + $nugetCommand = Get-Command 'nuget' -ErrorAction SilentlyContinue + } + + if ($nugetCommand) { + return $nugetCommand.Source + } + + $toolsDir = Join-Path $repoRoot '.tmp/tools' + $nugetPath = Join-Path $toolsDir 'nuget.exe' + if (-not (Test-Path $nugetPath)) { + New-Item -ItemType Directory -Path $toolsDir -Force | Out-Null + Invoke-WebRequest -Uri 'https://dist.nuget.org/win-x86-commandline/latest/nuget.exe' -OutFile $nugetPath + } + + return $nugetPath +} + function Resolve-MSBuildPath { $vswhere = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio\Installer\vswhere.exe' if (-not (Test-Path $vswhere)) { @@ -56,6 +76,7 @@ function Invoke-Step { } $cmake = Resolve-CommandPath 'cmake' +$nuget = Resolve-NuGetPath $pnpm = Resolve-CommandPath 'pnpm' $msbuild = Resolve-MSBuildPath $generator = 'Visual Studio 17 2022' @@ -76,6 +97,10 @@ Invoke-Step 'Build asar-fuses-bypass' { & $cmake --build $asarFusesBuildDir --config $Configuration } +Invoke-Step 'Restore NuGet packages' { + & $nuget restore $solutionPath -NonInteractive +} + Invoke-Step 'Build solution' { & $msbuild $solutionPath /m /p:Configuration=$Configuration '/p:Platform=Any CPU' /t:Build }