#!/bin/bash

## Not responsable if you open too many connections and overload target server. 

#Script to use wget faster; Extracts URLs from a X page and then downloads those links with N number of connections
#Example: ./fasterDL.sh https://genome.nyumc.org/results/external/tamucc/2015-04-03/fastq/ 4

#Will only run if there are 2 arguments
EXPECTED_ARGS=2
if [ $# -ne $EXPECTED_ARGS ]; then
	printf "Needs 2 arguments, URL and Number of connections\n"
	printf "Example: ./fasterDL.sh https://genome.nyumc.org/results/external/tamucc/2015-04-03/fastq/ 4\n"
	exit
fi

#Will only run if the directory 'outdir' does not exist
if [ -d "outdir" ]; then
	printf "Directory 'outdir' exists. Please move or remove before running this script\n"
	exit
fi

printf "Please be nice to the servers and don't go too fast!\n"

#extract list of URLs within the URL
python extract.py $1 > FASTERDL_14231431_URLS.txt

#create directory to store downloaded files
mkdir outdir

cat FASTERDL_14231431_URLS.txt | xargs -n 1 -P $2 wget -q -P outdir/