February 01, 2014

ChunkGet - Parallel download

Sharing a simple script that helps you to download large files faster in case if you have limited bandwidth per established connection.

#!/bin/bash
#
# chunkget v0.1
# ChunkGet - Fast file download in parallel chunks
#
# Andrey Arapov <andrey.arapov [at]="None" nixaid.com="None"> - 2013.11.27 - initial alpha release v0.1
#
#

URL=$1
CHUNKS=$2

[ -z "$1" ] && echo "Usage: $0 url [chunks]" && echo "Example: $0 http://example.org/file.iso 10" && exit 1

# Make sure that we located the file
AURL=$(curl -sI ${URL} |grep Location | awk '{print $2}'|sed 's/[[:space:]]//g')
if ! [ -z ${AURL} ]; then URL=${AURL} ; echo "Link switched to ${URL}"; fi

# Getting the output filename in case if it was not specified
[ -z ${OUTPUT} ] && OUTPUT=$(echo ${URL} |sed 's/.*\///g')

# Getting the length of a file
LENGTH=$(curl -sI ${URL} | grep Content-Length | awk '{print $2}'|sed 's/[[:space:]]//g')
if [ -z ${LENGTH} ]; then echo "Could not get the length of the file." ; exit 1 ; fi

# Calculating the blocksize
[ -z ${CHUNKS} ] && CHUNKS=10
BLOCKSIZE=$[ ${LENGTH}/${CHUNKS} ]

# Processing chunks
for i in $(seq 0 $[ ${CHUNKS}-1 ]); do
  # If we are at the last chunk, then we set end range to the length of the file, otherwise we will calculate the range based on a current chunk
  [ ${i} -eq $[ ${CHUNKS}-1 ] ] && RANGE=$[ ${BLOCKSIZE} * ${i} ]-$[ ${LENGTH} ] || RANGE=$[ ${BLOCKSIZE} * ${i} ]-$[ ${BLOCKSIZE} * $[${i}+1] ] ;

  # Start download threads (via cron) with calculated ranges
  echo "curl -s --range $RANGE ${URL} |dd of=${OUTPUT} bs=${BLOCKSIZE} seek=${i} conv=notrunc status=noxfer >/dev/null 2>&1" | at now ;
done

echo "Run 'atq' to see running threads. Run 'killall curl' to stop downloading (NOTE: Check processes before in order not to kill unwanted process)"