diff --git a/DepotDownloader/ConfigStore.cs b/DepotDownloader/ConfigStore.cs new file mode 100644 index 000000000..331419bc1 --- /dev/null +++ b/DepotDownloader/ConfigStore.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using ProtoBuf; +using System.IO; +using System.IO.Compression; + +namespace DepotDownloader +{ + [ProtoContract] + class ConfigStore + { + [ProtoMember(1)] + public Dictionary LastManifests { get; private set; } + + [ProtoMember(3, IsRequired=false)] + public Dictionary SentryData { get; private set; } + + string FileName = null; + + ConfigStore() + { + LastManifests = new Dictionary(); + SentryData = new Dictionary(); + } + + static bool Loaded + { + get { return TheConfig != null; } + } + + public static ConfigStore TheConfig = null; + + public static void LoadFromFile(string filename) + { + if (Loaded) + throw new Exception("Config already loaded"); + + if (File.Exists(filename)) + { + using (FileStream fs = File.Open(filename, FileMode.Open)) + using (DeflateStream ds = new DeflateStream(fs, CompressionMode.Decompress)) + TheConfig = ProtoBuf.Serializer.Deserialize(ds); + } + else + { + TheConfig = new ConfigStore(); + } + + TheConfig.FileName = filename; + } + + public static void Save() + { + if (!Loaded) + throw new Exception("Saved config before loading"); + + using (FileStream fs = File.Open(TheConfig.FileName, FileMode.Create)) + using (DeflateStream ds = new DeflateStream(fs, CompressionMode.Compress)) + ProtoBuf.Serializer.Serialize(ds, TheConfig); + } + } +} diff --git a/DepotDownloader/ContentDownloader.cs b/DepotDownloader/ContentDownloader.cs index bcf349db8..5bfc5bc51 100644 --- a/DepotDownloader/ContentDownloader.cs +++ b/DepotDownloader/ContentDownloader.cs @@ -436,13 +436,52 @@ static DepotDownloadInfo GetDepotInfo(uint depotId, uint appId, string branch) private class ChunkMatch { - public ChunkMatch(ProtoManifest.ChunkData oldChunk, DepotManifest.ChunkData newChunk) + public ChunkMatch(ProtoManifest.ChunkData oldChunk, ProtoManifest.ChunkData newChunk) { OldChunk = oldChunk; NewChunk = newChunk; } public ProtoManifest.ChunkData OldChunk { get; private set; } - public DepotManifest.ChunkData NewChunk { get; private set; } + public ProtoManifest.ChunkData NewChunk { get; private set; } + } + + private static List CollectCDNClientsForDepot(DepotDownloadInfo depot) + { + var cdnClients = new List(); + CDNClient initialClient = new CDNClient(steam3.steamClient, depot.id, steam3.AppTickets[depot.id], depot.depotKey); + var cdnServers = initialClient.FetchServerList(cellId: (uint)Config.CellID); + + if (cdnServers.Count == 0) + { + Console.WriteLine("\nUnable to find any content servers for depot {0} - {1}", depot.id, depot.contentName); + return null; + } + + // Grab up to the first eight server in the allegedly best-to-worst order from Steam + Enumerable.Range(0, Math.Min(cdnServers.Count, Config.MaxServers)).ToList().ForEach(s => + { + CDNClient c; + if( s == 0 ) + { + c = initialClient; + } + else + { + c = new CDNClient(steam3.steamClient, depot.id, steam3.AppTickets[depot.id], depot.depotKey); + } + + try + { + c.Connect(cdnServers[s]); + cdnClients.Add(c); + } + catch + { + Console.WriteLine("\nFailed to connect to content server {0}. Remaining content servers for depot: {1}.", cdnServers[s], cdnServers.Count - s - 1); + } + }); + + return cdnClients; } private static void DownloadSteam3( List depots ) @@ -458,70 +497,84 @@ private static void DownloadSteam3( List depots ) Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName); Console.Write("Finding content servers..."); - var cdnClients = new List(); - CDNClient initialClient = new CDNClient(steam3.steamClient, depot.id, steam3.AppTickets[depot.id], depot.depotKey); - var cdnServers = initialClient.FetchServerList(cellId: (uint)Config.CellID); + List cdnClients = null; - if (cdnServers.Count == 0) + Console.WriteLine(" Done!"); + + ProtoManifest oldProtoManifest = null; + ProtoManifest newProtoManifest = null; + string configDir = Path.Combine(depot.installDir, CONFIG_DIR); + + ulong lastManifestId = INVALID_MANIFEST_ID; + ConfigStore.TheConfig.LastManifests.TryGetValue(depot.id, out lastManifestId); + + // In case we have an early exit, this will force equiv of verifyall next run. + ConfigStore.TheConfig.LastManifests[depot.id] = INVALID_MANIFEST_ID; + ConfigStore.Save(); + + if (lastManifestId != INVALID_MANIFEST_ID) { - Console.WriteLine("\nUnable to find any content servers for depot {0} - {1}", depot.id, depot.contentName); - continue; + var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId)); + if (File.Exists(oldManifestFileName)) + oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName); } - // Grab up to the first eight server in the allegedly best-to-worst order from Steam - Enumerable.Range(0, Math.Min(cdnServers.Count, Config.MaxServers)).ToList().ForEach(s => + if (lastManifestId == depot.manifestId && oldProtoManifest != null) { - CDNClient c; - if( s == 0 ) - { - c = initialClient; - } - else + newProtoManifest = oldProtoManifest; + Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); + } + else + { + var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId)); + if (newManifestFileName != null) { - c = new CDNClient(steam3.steamClient, depot.id, steam3.AppTickets[depot.id], depot.depotKey); + newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName); } - try + if (newProtoManifest != null) { - c.Connect(cdnServers[s]); - cdnClients.Add(c); + Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); } - catch + else { - Console.WriteLine("\nFailed to connect to content server {0}. Remaining content servers for depot: {1}.", cdnServers[s], cdnServers.Count - s - 1); - } - }); + Console.Write("Downloading depot manifest..."); - Console.WriteLine(" Done!"); - Console.Write("Downloading depot manifest..."); + DepotManifest depotManifest = null; - DepotManifest depotManifest = null; - foreach (var c in cdnClients) - { - try - { - depotManifest = c.DownloadManifest(depot.manifestId); - break; - } - catch (WebException) { } - } + cdnClients = CollectCDNClientsForDepot(depot); - if ( depotManifest == null ) - { - Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id); - return; - } + foreach (var c in cdnClients) + { + try + { + depotManifest = c.DownloadManifest(depot.manifestId); + break; + } + catch (WebException) { } + } - Console.WriteLine(" Done!"); + if (depotManifest == null) + { + Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id); + return; + } + + newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId); + newProtoManifest.SaveToFile(newManifestFileName); + + Console.WriteLine(" Done!"); + } + } - depotManifest.Files.Sort((x, y) => { return x.FileName.CompareTo(y.FileName); }); + newProtoManifest.Files.Sort((x, y) => { return x.FileName.CompareTo(y.FileName); }); if (Config.DownloadManifestOnly) { StringBuilder manifestBuilder = new StringBuilder(); string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}.txt", depot.id)); - foreach (var file in depotManifest.Files) + foreach (var file in newProtoManifest.Files) { if (file.Flags.HasFlag(EDepotFileFlag.Directory)) continue; @@ -535,31 +588,10 @@ private static void DownloadSteam3( List depots ) ulong complete_download_size = 0; ulong size_downloaded = 0; - string configDir = Path.Combine(depot.installDir, CONFIG_DIR); string stagingDir = Path.Combine(depot.installDir, STAGING_DIR); - ProtoManifest oldProtoManifest = null; - ProtoManifest newProtoManifest = null; - - { - var oldManifestFileName = Directory.GetFiles(configDir, string.Format("{0}.bin", depot.id)).OrderByDescending(x => File.GetLastWriteTimeUtc(x)).FirstOrDefault(); - if (oldManifestFileName != null) - { - if (!Config.VerifyAll) - { - oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName); - } - - // Delete this regardless. If we finish successfully, we'll write the new one. - File.Delete(oldManifestFileName); - } - } - - depotManifest.Files.RemoveAll((x) => !TestIsFileIncluded(x.FileName)); - - newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId); - + // Pre-process - depotManifest.Files.ForEach(file => + newProtoManifest.Files.ForEach(file => { var fileFinalPath = Path.Combine(depot.installDir, file.FileName); var fileStagingPath = Path.Combine(stagingDir, file.FileName); @@ -581,11 +613,14 @@ private static void DownloadSteam3( List depots ) var rand = new Random(); - depotManifest.Files.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)) + newProtoManifest.Files.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)) .AsParallel().WithDegreeOfParallelism(Config.MaxDownloads) .ForAll(file => { - var clientIndex = rand.Next(0, cdnClients.Count); + if (!TestIsFileIncluded(file.FileName)) + { + return; + } string fileFinalPath = Path.Combine(depot.installDir, file.FileName); string fileStagingPath = Path.Combine(stagingDir, file.FileName); @@ -597,14 +632,14 @@ private static void DownloadSteam3( List depots ) } FileStream fs = null; - List neededChunks; + List neededChunks; FileInfo fi = new FileInfo(fileFinalPath); if (!fi.Exists) { // create new file. need all chunks fs = File.Create(fileFinalPath); fs.SetLength((long)file.TotalSize); - neededChunks = new List(file.Chunks); + neededChunks = new List(file.Chunks); } else { @@ -617,9 +652,9 @@ private static void DownloadSteam3( List depots ) if (oldManifestFile != null) { - neededChunks = new List(); + neededChunks = new List(); - if (!oldManifestFile.FileHash.SequenceEqual(file.FileHash)) + if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash)) { // we have a version of this file, but it doesn't fully match what we want @@ -686,17 +721,50 @@ private static void DownloadSteam3( List depots ) } } + int cdnClientIndex = 0; + if (neededChunks.Count > 0 && cdnClients == null) + { + // If we didn't need to connect to get manifests, connect now. + cdnClients = CollectCDNClientsForDepot(depot); + cdnClientIndex = rand.Next(0, cdnClients.Count); + } + foreach (var chunk in neededChunks) { string chunkID = Util.EncodeHexString(chunk.ChunkID); CDNClient.DepotChunk chunkData = null; - int idx = clientIndex; + int idx = cdnClientIndex; while (true) { try { - chunkData = cdnClients[idx].DownloadDepotChunk(chunk); +#if true + // The only way that SteamKit exposes to get a DepotManifest.ChunkData instance is to download a new manifest. + // We only want to download manifests that we don't already have, so we'll have to improvise... + + // internal ChunkData( byte[] id, byte[] checksum, ulong offset, uint comp_length, uint uncomp_length ) + System.Reflection.ConstructorInfo ctor = typeof(DepotManifest.ChunkData).GetConstructor( + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.CreateInstance | System.Reflection.BindingFlags.Instance, + null, + new[] { typeof(byte[]), typeof(byte[]), typeof(ulong), typeof(uint), typeof(uint) }, + null); + var data = (DepotManifest.ChunkData)ctor.Invoke( + new object[] { + chunk.ChunkID, chunk.Checksum, chunk.Offset, chunk.CompressedLength, chunk.UncompressedLength + }); + +#else + // Next SteamKit version after 1.5.0 will support this. + // Waiting for it to be in the NuGet repo. + DepotManifest.ChunkData data = new DepotManifest.ChunkData(); + data.ChunkID = chunk.ChunkID; + data.Checksum = chunk.Checksum; + data.Offset = chunk.Offset; + data.CompressedLength = chunk.CompressedLength; + data.UncompressedLength = chunk.UncompressedLength; +#endif + chunkData = cdnClients[idx].DownloadDepotChunk(data); break; } catch @@ -704,7 +772,7 @@ private static void DownloadSteam3( List depots ) if (++idx >= cdnClients.Count) idx = 0; - if (idx == clientIndex) + if (idx == cdnClientIndex) break; } } @@ -731,7 +799,8 @@ private static void DownloadSteam3( List depots ) Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, fileFinalPath); }); - newProtoManifest.SaveToFile(Path.Combine(configDir, string.Format("{0}.bin", depot.id))); + ConfigStore.TheConfig.LastManifests[depot.id] = depot.manifestId; + ConfigStore.Save(); Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed); } diff --git a/DepotDownloader/DepotDownloader.csproj b/DepotDownloader/DepotDownloader.csproj index 0dabca385..4a4031262 100644 --- a/DepotDownloader/DepotDownloader.csproj +++ b/DepotDownloader/DepotDownloader.csproj @@ -81,6 +81,7 @@ + diff --git a/DepotDownloader/Program.cs b/DepotDownloader/Program.cs index 1c73de1c7..c27ec4869 100644 --- a/DepotDownloader/Program.cs +++ b/DepotDownloader/Program.cs @@ -19,6 +19,8 @@ static void Main( string[] args ) DebugLog.Enabled = false; + ConfigStore.LoadFromFile(Path.Combine(Environment.CurrentDirectory, "DepotDownloader.config")); + bool bDumpManifest = HasParameter( args, "-manifest-only" ); uint appId = GetParameter( args, "-app", ContentDownloader.INVALID_APP_ID ); uint depotId = GetParameter( args, "-depot", ContentDownloader.INVALID_DEPOT_ID ); diff --git a/DepotDownloader/ProtoManifest.cs b/DepotDownloader/ProtoManifest.cs index 3ac5677f1..edc002e45 100644 --- a/DepotDownloader/ProtoManifest.cs +++ b/DepotDownloader/ProtoManifest.cs @@ -12,11 +12,13 @@ namespace DepotDownloader class ProtoManifest { // Proto ctor - private ProtoManifest() { } - - public ProtoManifest(DepotManifest sourceManifest, ulong id) + private ProtoManifest() { Files = new List(); + } + + public ProtoManifest(DepotManifest sourceManifest, ulong id) : this() + { sourceManifest.Files.ForEach(f => Files.Add(new FileData(f))); ID = id; } @@ -25,10 +27,13 @@ public ProtoManifest(DepotManifest sourceManifest, ulong id) public class FileData { // Proto ctor - private FileData() { } - public FileData(DepotManifest.FileData sourceData) + private FileData() { Chunks = new List(); + } + + public FileData(DepotManifest.FileData sourceData) : this() + { FileName = sourceData.FileName; sourceData.Chunks.ForEach(c => Chunks.Add(new ChunkData(c))); Flags = sourceData.Flags; diff --git a/DepotDownloader/Steam3Session.cs b/DepotDownloader/Steam3Session.cs index ea4abae7a..8af2bbc62 100644 --- a/DepotDownloader/Steam3Session.cs +++ b/DepotDownloader/Steam3Session.cs @@ -89,9 +89,16 @@ public Steam3Session( SteamUser.LogOnDetails details ) if ( authenticatedUser ) { FileInfo fi = new FileInfo(String.Format("{0}.sentryFile", logonDetails.Username)); - if (fi.Exists && fi.Length > 0) + if (ConfigStore.TheConfig.SentryData != null && ConfigStore.TheConfig.SentryData.ContainsKey(logonDetails.Username)) { - logonDetails.SentryFileHash = Util.SHAHash(File.ReadAllBytes(fi.FullName)); + logonDetails.SentryFileHash = Util.SHAHash(ConfigStore.TheConfig.SentryData[logonDetails.Username]); + } + else if (fi.Exists && fi.Length > 0) + { + var sentryData = File.ReadAllBytes(fi.FullName); + logonDetails.SentryFileHash = Util.SHAHash(sentryData); + ConfigStore.TheConfig.SentryData[logonDetails.Username] = sentryData; + ConfigStore.Save(); } } @@ -426,7 +433,9 @@ private void UpdateMachineAuthCallback(SteamUser.UpdateMachineAuthCallback machi byte[] hash = Util.SHAHash(machineAuth.Data); Console.WriteLine("Got Machine Auth: {0} {1} {2} {3}", machineAuth.FileName, machineAuth.Offset, machineAuth.BytesToWrite, machineAuth.Data.Length, hash); - File.WriteAllBytes( String.Format("{0}.sentryFile", logonDetails.Username), machineAuth.Data ); + ConfigStore.TheConfig.SentryData[logonDetails.Username] = machineAuth.Data; + ConfigStore.Save(); + var authResponse = new SteamUser.MachineAuthDetails { BytesWritten = machineAuth.BytesToWrite, diff --git a/DepotDownloader/Util.cs b/DepotDownloader/Util.cs index df99fd2e4..60369688f 100644 --- a/DepotDownloader/Util.cs +++ b/DepotDownloader/Util.cs @@ -96,12 +96,12 @@ public static string ReadPassword() } // Validate a file against Steam3 Chunk data - public static List ValidateSteam3FileChecksums(FileStream fs, DepotManifest.ChunkData[] chunkdata) + public static List ValidateSteam3FileChecksums(FileStream fs, ProtoManifest.ChunkData[] chunkdata) { - var neededChunks = new List(); + var neededChunks = new List(); int read; - foreach (DepotManifest.ChunkData data in chunkdata) + foreach (var data in chunkdata) { byte[] chunk = new byte[data.UncompressedLength]; fs.Seek((long)data.Offset, SeekOrigin.Begin);