#!/bin/bash

dir="$PWD"
set -e
git config --global user.name "${GIT_NAME}"
git config --global user.email "${GIT_EMAIL}"
git config --global push.default simple
tmp="$dir/tmp"
mkdir $tmp
cd $tmp
wget http://profile.gigaset.net/logs.tgz && tar -xzf logs.tgz
cd log
for f in ./*;do {
echo "Working... for $f"
sed -i '/200\|206\|302\|301/!d;s/\sHTTP.*//g;s/.*\s/http:\/\/profile\.gigaset\.net/g;/http:\/\/profile\.gigaset\.net\"\"/d;s/?.*//g;s/\.net\/device/\.net\/chagall/g;s/^\.$//g' $f
echo "Remove duplicates for $f"
awk '!seen[$0]++' $f>>$tmp/tmp
};done
echo "Remove all duplicates"
awk '!seen[$0]++' $tmp/tmp $dir/urllist >$tmp/final
echo "Remove useless stuff..."
sed -i '/200\|206\|302\|301/!d;s/\sHTTP.*//g;s/.*\s/http:\/\/profile\.gigaset\.net/g;/http:\/\/profile\.gigaset\.net\"\"/d;s/?.*//g;s/\.net\/device/\.net\/chagall/g;s/^\.$//g' $tmp/final
echo "Remove all duplicates"
awk '!seen[$0]++' $tmp/final >$tmp/finall

mv $tmp/finall $dir/urllist

git clone https://${GH_TOKEN}@github.com/danog/gigaclone.git $tmp/git
cd $tmp/git
cp $dir/urllist .
git add -A

git commit -m "Updated url list"
git push origin master &>/dev/null

cd $dir
echo "Clean up."
rm -fr $tmp