Skip to content

Commit

Permalink
Big speed improvements for some cases:
Browse files Browse the repository at this point in the history 
- Store all manifests separately, including excluded file, rather than only list of last-downloaded.
- Don't redownload manifests we have.
- Don't connect to content servers if no manifest to download and no chunks needed.
- Don't connect to content servers until needing chunks if already having manifest.
  • Loading branch information
@psychonic
psychonic committed Nov 14, 2013
1 parent d59f352 commit d9cec26
Show file tree
Hide file tree
Showing 7 changed files with 238 additions and 88 deletions.
64 changes: 64 additions & 0 deletions DepotDownloader/ConfigStore.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using ProtoBuf;
using System.IO;
using System.IO.Compression;

namespace DepotDownloader
{
[ProtoContract]
class ConfigStore
{
[ProtoMember(1)]
public Dictionary<uint, ulong> LastManifests { get; private set; }

[ProtoMember(3, IsRequired=false)]
public Dictionary<string, byte[]> SentryData { get; private set; }

string FileName = null;

ConfigStore()
{
LastManifests = new Dictionary<uint, ulong>();
SentryData = new Dictionary<string, byte[]>();
}

static bool Loaded
{
get { return TheConfig != null; }
}

public static ConfigStore TheConfig = null;

public static void LoadFromFile(string filename)
{
if (Loaded)
throw new Exception("Config already loaded");

if (File.Exists(filename))
{
using (FileStream fs = File.Open(filename, FileMode.Open))
using (DeflateStream ds = new DeflateStream(fs, CompressionMode.Decompress))
TheConfig = ProtoBuf.Serializer.Deserialize<ConfigStore>(ds);
}
else
{
TheConfig = new ConfigStore();
}

TheConfig.FileName = filename;
}

public static void Save()
{
if (!Loaded)
throw new Exception("Saved config before loading");

using (FileStream fs = File.Open(TheConfig.FileName, FileMode.Create))
using (DeflateStream ds = new DeflateStream(fs, CompressionMode.Compress))
ProtoBuf.Serializer.Serialize<ConfigStore>(ds, TheConfig);
}
}
}
223 changes: 146 additions & 77 deletions DepotDownloader/ContentDownloader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -436,13 +436,52 @@ static DepotDownloadInfo GetDepotInfo(uint depotId, uint appId, string branch)

private class ChunkMatch
{
public ChunkMatch(ProtoManifest.ChunkData oldChunk, DepotManifest.ChunkData newChunk)
public ChunkMatch(ProtoManifest.ChunkData oldChunk, ProtoManifest.ChunkData newChunk)
{
OldChunk = oldChunk;
NewChunk = newChunk;
}
public ProtoManifest.ChunkData OldChunk { get; private set; }
public DepotManifest.ChunkData NewChunk { get; private set; }
public ProtoManifest.ChunkData NewChunk { get; private set; }
}

private static List<CDNClient> CollectCDNClientsForDepot(DepotDownloadInfo depot)
{
var cdnClients = new List<CDNClient>();
CDNClient initialClient = new CDNClient(steam3.steamClient, depot.id, steam3.AppTickets[depot.id], depot.depotKey);
var cdnServers = initialClient.FetchServerList(cellId: (uint)Config.CellID);

if (cdnServers.Count == 0)
{
Console.WriteLine("\nUnable to find any content servers for depot {0} - {1}", depot.id, depot.contentName);
return null;
}

// Grab up to the first eight server in the allegedly best-to-worst order from Steam
Enumerable.Range(0, Math.Min(cdnServers.Count, Config.MaxServers)).ToList().ForEach(s =>
{
CDNClient c;
if( s == 0 )
{
c = initialClient;
}
else
{
c = new CDNClient(steam3.steamClient, depot.id, steam3.AppTickets[depot.id], depot.depotKey);
}

try
{
c.Connect(cdnServers[s]);
cdnClients.Add(c);
}
catch
{
Console.WriteLine("\nFailed to connect to content server {0}. Remaining content servers for depot: {1}.", cdnServers[s], cdnServers.Count - s - 1);
}
});

return cdnClients;
}

private static void DownloadSteam3( List<DepotDownloadInfo> depots )
Expand All @@ -458,70 +497,84 @@ private static void DownloadSteam3( List<DepotDownloadInfo> depots )
Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName);
Console.Write("Finding content servers...");

var cdnClients = new List<CDNClient>();
CDNClient initialClient = new CDNClient(steam3.steamClient, depot.id, steam3.AppTickets[depot.id], depot.depotKey);
var cdnServers = initialClient.FetchServerList(cellId: (uint)Config.CellID);
List<CDNClient> cdnClients = null;

if (cdnServers.Count == 0)
Console.WriteLine(" Done!");

ProtoManifest oldProtoManifest = null;
ProtoManifest newProtoManifest = null;
string configDir = Path.Combine(depot.installDir, CONFIG_DIR);

ulong lastManifestId = INVALID_MANIFEST_ID;
ConfigStore.TheConfig.LastManifests.TryGetValue(depot.id, out lastManifestId);

// In case we have an early exit, this will force equiv of verifyall next run.
ConfigStore.TheConfig.LastManifests[depot.id] = INVALID_MANIFEST_ID;
ConfigStore.Save();

if (lastManifestId != INVALID_MANIFEST_ID)
{
Console.WriteLine("\nUnable to find any content servers for depot {0} - {1}", depot.id, depot.contentName);
continue;
var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId));
if (File.Exists(oldManifestFileName))
oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName);
}

// Grab up to the first eight server in the allegedly best-to-worst order from Steam
Enumerable.Range(0, Math.Min(cdnServers.Count, Config.MaxServers)).ToList().ForEach(s =>
if (lastManifestId == depot.manifestId && oldProtoManifest != null)
{
CDNClient c;
if( s == 0 )
{
c = initialClient;
}
else
newProtoManifest = oldProtoManifest;
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
}
else
{
var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId));
if (newManifestFileName != null)
{
c = new CDNClient(steam3.steamClient, depot.id, steam3.AppTickets[depot.id], depot.depotKey);
newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName);
}

try
if (newProtoManifest != null)
{
c.Connect(cdnServers[s]);
cdnClients.Add(c);
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
}
catch
else
{
Console.WriteLine("\nFailed to connect to content server {0}. Remaining content servers for depot: {1}.", cdnServers[s], cdnServers.Count - s - 1);
}
});
Console.Write("Downloading depot manifest...");

Console.WriteLine(" Done!");
Console.Write("Downloading depot manifest...");
DepotManifest depotManifest = null;

DepotManifest depotManifest = null;
foreach (var c in cdnClients)
{
try
{
depotManifest = c.DownloadManifest(depot.manifestId);
break;
}
catch (WebException) { }
}
cdnClients = CollectCDNClientsForDepot(depot);

if ( depotManifest == null )
{
Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id);
return;
}
foreach (var c in cdnClients)
{
try
{
depotManifest = c.DownloadManifest(depot.manifestId);
break;
}
catch (WebException) { }
}

Console.WriteLine(" Done!");
if (depotManifest == null)
{
Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id);
return;
}

newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId);
newProtoManifest.SaveToFile(newManifestFileName);

Console.WriteLine(" Done!");
}
}

depotManifest.Files.Sort((x, y) => { return x.FileName.CompareTo(y.FileName); });
newProtoManifest.Files.Sort((x, y) => { return x.FileName.CompareTo(y.FileName); });

if (Config.DownloadManifestOnly)
{
StringBuilder manifestBuilder = new StringBuilder();
string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}.txt", depot.id));

foreach (var file in depotManifest.Files)
foreach (var file in newProtoManifest.Files)
{
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
continue;
Expand All @@ -535,31 +588,10 @@ private static void DownloadSteam3( List<DepotDownloadInfo> depots )

ulong complete_download_size = 0;
ulong size_downloaded = 0;
string configDir = Path.Combine(depot.installDir, CONFIG_DIR);
string stagingDir = Path.Combine(depot.installDir, STAGING_DIR);
ProtoManifest oldProtoManifest = null;
ProtoManifest newProtoManifest = null;

{
var oldManifestFileName = Directory.GetFiles(configDir, string.Format("{0}.bin", depot.id)).OrderByDescending(x => File.GetLastWriteTimeUtc(x)).FirstOrDefault();
if (oldManifestFileName != null)
{
if (!Config.VerifyAll)
{
oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName);
}

// Delete this regardless. If we finish successfully, we'll write the new one.
File.Delete(oldManifestFileName);
}
}

depotManifest.Files.RemoveAll((x) => !TestIsFileIncluded(x.FileName));

newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId);


// Pre-process
depotManifest.Files.ForEach(file =>
newProtoManifest.Files.ForEach(file =>
{
var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
var fileStagingPath = Path.Combine(stagingDir, file.FileName);
Expand All @@ -581,11 +613,14 @@ private static void DownloadSteam3( List<DepotDownloadInfo> depots )

var rand = new Random();

depotManifest.Files.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory))
newProtoManifest.Files.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory))
.AsParallel().WithDegreeOfParallelism(Config.MaxDownloads)
.ForAll(file =>
{
var clientIndex = rand.Next(0, cdnClients.Count);
if (!TestIsFileIncluded(file.FileName))
{
return;
}

string fileFinalPath = Path.Combine(depot.installDir, file.FileName);
string fileStagingPath = Path.Combine(stagingDir, file.FileName);
Expand All @@ -597,14 +632,14 @@ private static void DownloadSteam3( List<DepotDownloadInfo> depots )
}

FileStream fs = null;
List<DepotManifest.ChunkData> neededChunks;
List<ProtoManifest.ChunkData> neededChunks;
FileInfo fi = new FileInfo(fileFinalPath);
if (!fi.Exists)
{
// create new file. need all chunks
fs = File.Create(fileFinalPath);
fs.SetLength((long)file.TotalSize);
neededChunks = new List<DepotManifest.ChunkData>(file.Chunks);
neededChunks = new List<ProtoManifest.ChunkData>(file.Chunks);
}
else
{
Expand All @@ -617,9 +652,9 @@ private static void DownloadSteam3( List<DepotDownloadInfo> depots )

if (oldManifestFile != null)
{
neededChunks = new List<DepotManifest.ChunkData>();
neededChunks = new List<ProtoManifest.ChunkData>();

if (!oldManifestFile.FileHash.SequenceEqual(file.FileHash))
if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash))
{
// we have a version of this file, but it doesn't fully match what we want

Expand Down Expand Up @@ -686,25 +721,58 @@ private static void DownloadSteam3( List<DepotDownloadInfo> depots )
}
}

int cdnClientIndex = 0;
if (neededChunks.Count > 0 && cdnClients == null)
{
// If we didn't need to connect to get manifests, connect now.
cdnClients = CollectCDNClientsForDepot(depot);
cdnClientIndex = rand.Next(0, cdnClients.Count);
}

foreach (var chunk in neededChunks)
{
string chunkID = Util.EncodeHexString(chunk.ChunkID);

CDNClient.DepotChunk chunkData = null;
int idx = clientIndex;
int idx = cdnClientIndex;
while (true)
{
try
{
chunkData = cdnClients[idx].DownloadDepotChunk(chunk);
#if true
// The only way that SteamKit exposes to get a DepotManifest.ChunkData instance is to download a new manifest.
// We only want to download manifests that we don't already have, so we'll have to improvise...

// internal ChunkData( byte[] id, byte[] checksum, ulong offset, uint comp_length, uint uncomp_length )
System.Reflection.ConstructorInfo ctor = typeof(DepotManifest.ChunkData).GetConstructor(
System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.CreateInstance | System.Reflection.BindingFlags.Instance,
null,
new[] { typeof(byte[]), typeof(byte[]), typeof(ulong), typeof(uint), typeof(uint) },
null);
var data = (DepotManifest.ChunkData)ctor.Invoke(
new object[] {
chunk.ChunkID, chunk.Checksum, chunk.Offset, chunk.CompressedLength, chunk.UncompressedLength
});

#else
// Next SteamKit version after 1.5.0 will support this.
// Waiting for it to be in the NuGet repo.
DepotManifest.ChunkData data = new DepotManifest.ChunkData();
data.ChunkID = chunk.ChunkID;
data.Checksum = chunk.Checksum;
data.Offset = chunk.Offset;
data.CompressedLength = chunk.CompressedLength;
data.UncompressedLength = chunk.UncompressedLength;
#endif
chunkData = cdnClients[idx].DownloadDepotChunk(data);
break;
}
catch
{
if (++idx >= cdnClients.Count)
idx = 0;

if (idx == clientIndex)
if (idx == cdnClientIndex)
break;
}
}
Expand All @@ -731,7 +799,8 @@ private static void DownloadSteam3( List<DepotDownloadInfo> depots )
Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, fileFinalPath);
});

newProtoManifest.SaveToFile(Path.Combine(configDir, string.Format("{0}.bin", depot.id)));
ConfigStore.TheConfig.LastManifests[depot.id] = depot.manifestId;
ConfigStore.Save();

Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed);
}
Expand Down
Loading

0 comments on commit d9cec26

Please sign in to comment.