danog/gigaclone
1
0
Fork 0
mirror of https://github.com/danog/gigaclone.git synced 2024-11-26 20:14:58 +01:00
Code Issues Projects Releases Wiki Activity

FIx urllist

Browse Source
This commit is contained in:
Daniil Gentili 2016-04-08 21:42:41 +02:00
parent 4215430820
commit 7d6e60a96a
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
mkurllist
View File

@ -24,12 +24,13 @@ cd $tmp
rdupes() {
[ "$1" != "" ] && {
echo "Remove all duplicates for $*"
sed -i 's/\/\//\//g;s/\/$//g' $dir/urllist
awk '!seen[$0]++' $* $dir/urllist >$dir/final
mv $dir/final $dir/urllist
rm $*
}
sed -i 's/\/\//\//g;s/\/$//g;s/http:\//http:\/\//g;s/http:\/\/\//http:\/\//g' $dir/urllist
for dl in $(cat $dir/urllist); do {
a=$(curl -w "%{url_effective}\n" -L -f -s -I -S "$dl" -o /dev/null)
[ $? = "0" ] && echo "$a" >> $dir/final