gigaclone/.travis.gigaclone

#!/bin/bash

dir="$PWD"

git config --global user.name "${GIT_NAME}"
git config --global user.email "${GIT_EMAIL}"
git config --global push.default simple
tmp="$dir/tmp"
mkdir $tmp
cd $tmp
rdupes() {
echo "Remove all duplicates"
awk '!seen[$0]++' $* $dir/urllist >$dir/final
mv $dir/final $dir/urllist
rm $*
cd $dir
rm -rf $tmp
mkdir $tmp
cd $tmp
}
extractlog() {
  
tar -xzf logs.tgz
cd log

for f in ./*;do {
echo "Working... for $f"
sed -i '/200\|206\|302\|301/!d;s/\sHTTP.*//g;s/.*\s/http:\/\/profile\.gigaset\.net/g;/http:\/\/profile\.gigaset\.net\"\"/d;s/?.*//g;s/\.net\/device/\.net\/chagall/g;s/^\.$//g' $f
echo "Remove duplicates for $f"
awk '!seen[$0]++' $f>>$tmp/tmp
};done
}

extractbin() {
cd $tmp
echo -n wh
for f in $(grep "\.bin" $dir/urllist);do {
echo -n "e" >&2
file=$(wget -qO- "$f" | strings)
echo "$file" | sed '/http:\/\//!d;/profile.gigaset.net\|update.gigaset.net/!d;s/.*http:\/\//http:\/\//g;s/update\.gigaset/profile\.gigaset/g'

baseurl="$(echo "$f" | sed 's/[^/]*$//')"

url=$(echo "$file" | sed '/http/d;/\.bin/!d')

for e in $url;do echo "$baseurl$e";done

}; done >> $tmp/tmp
}

checkurl() {
echo -n "Turn down for wh"
for dl in $(cat $dir/urllist); do {
echo -n "a" >&2
wget -q "$dl" || continue
curl -w "%{url_effective}\n" -L -s -I -S "$dl" -o /dev/null
};done | grep -v "HTTP" | awk '!seen[$0]++' >$dir/check
mv $dir/check $dir/urllist
rm .check
}

tree() {
mkdir -p $tmp/test
cd $tmp/test
for f in $(sed 's/http:\/\/profile.gigaset.net\///g;s/^\/*//g' $dir/urllist);do {
mkdir -p $f
echo -n "a"
};done
tree -T "profile.gigaset.net" -C -H http://profile.gigaset.net -o $dir/tree.html

}

wget http://profile.gigaset.net/logs.tgz 
[ "$(md5sum $tmp/logs.tgz)" != "$(cat $dir/.md5sum)" ] && {
extractlog
extractbin
md5sum $tmp/logs.tgz >$dir/.md5sum
rdupes $tmp/tmp
} || rm $dir/logs.tgz
cd $dir
[ -f .check ] && checkurl || touch .check
tree
cd $dir
rm -rf $tmp/
mkdir $tmp

git clone https://${GH_TOKEN}@github.com/danog/gigaclone.git $tmp/git
cd $tmp/git

cp $dir/* .
git add -A
git commit -m "Updated url list"
git push origin master &>/dev/null

git checkout gh-pages
cp $dir/tree.html .
git add -A
git commit -m "Updated dir tree"
git push origin gh-pages &>/dev/null

cd $dir
echo "Clean up."
rm -fr $tmp