Does download progress work well with asyncmap?

[navidcy]

I'm running a script and I get a download progress 100% but still keeps downloading for long time.

I'm wondering whether

ClimaOcean.jl/src/DataWrangling/DataWrangling.jl

Lines 26 to 54 in 7ddb30b

	function download_progress(total, now; filename="")
	messages = 10
	if total > 0
	fraction = now / total
	if fraction < 1 / messages && next_fraction[] == 0
	@info @sprintf("Downloading %s (size: %s)...", filename, pretty_filesize(total))
	next_fraction[] = 1 / messages
	download_start_time[] = time_ns()
	end
	if fraction > next_fraction[]
	elapsed = 1e-9 * (time_ns() - download_start_time[])
	msg = @sprintf(" ... downloaded %s (%d%% complete, %s)", pretty_filesize(now),
	100fraction, prettytime(elapsed))
	@info msg
	next_fraction[] = next_fraction[] + 1 / messages
	end
	else
	if now > 0 && next_fraction[] == 0
	@info "Downloading $filename..."
	next_fraction[] = 1 / messages
	download_start_time[] = time_ns()
	end
	end
	return nothing
	end

works OK with asyncmap, e.g., in

ClimaOcean.jl/src/DataWrangling/ECCO/ECCO_metadata.jl

Lines 234 to 255 in 7ddb30b

	asyncmap(metadata, ntasks=10) do metadatum # Distribute the download among tasks
	fileurl = metadata_url(url, metadatum)
	filepath = metadata_path(metadatum)
	if !isfile(filepath)
	instructions_msg = "\n See ClimaOcean.jl/src/ECCO/README.md for instructions."
	if isnothing(username)
	msg = "Could not find the ECCO_PASSWORD environment variable. \
	See ClimaOcean.jl/src/ECCO/README.md for instructions on obtaining \
	and setting your ECCO_USERNAME and ECCO_PASSWORD." * instructions_msg
	throw(ArgumentError(msg))
	elseif isnothing(password)
	msg = "Could not find the ECCO_PASSWORD environment variable. \
	See ClimaOcean.jl/src/ECCO/README.md for instructions on obtaining \
	and setting your ECCO_USERNAME and ECCO_PASSWORD." * instructions_msg
	throw(ArgumentError(msg))
	end
	Downloads.download(fileurl, filepath; downloader, progress=download_progress)
	end
	end

[glwagner]

Can you test with ntasks=1 to see?