#!/bin/bash dir="$PWD" set -e git config --global user.name "${GIT_NAME}" git config --global user.email "${GIT_EMAIL}" git config --global push.default simple tmp="$dir/tmp" mkdir $tmp cd $tmp wget http://profile.gigaset.net/logs.tgz && tar -xzf logs.tgz cd log for f in ./*;do { echo "Working... for $f" sed -i '/200\|206\|302\|301/!d;s/\sHTTP.*//g;s/.*\s/http:\/\/profile\.gigaset\.net/g;/http:\/\/profile\.gigaset\.net\"\"/d;s/?.*//g;s/\.net\/device/\.net\/chagall/g;s/^\.$//g' $f echo "Remove duplicates for $f" awk '!seen[$0]++' $f>>$tmp/tmp };done echo "Remove all duplicates" awk '!seen[$0]++' $tmp/tmp $dir/urllist >$tmp/final for f in $(grep "\.bin" final);do { echo -n "Working... for $f " >&2 file=$(wget -qO- $f | strings) echo -n "Working... for $f " >&2 echo "$file" | sed '/http:\/\//!d;/profile.gigaset.net\|update.gigaset.net/!d;s/.*http:\/\//http:\/\//g;s/update\.gigaset/profile\.gigaset/g' baseurl="$(echo "$f" | sed 's/[^/]*$//')" url=$(echo "$file" | sed '/http/d;/\.bin/!d') for e in $url;do echo "$baseurl$e";done }; done >> $dir/final awk '!seen[$0]++' $dir/final > $dir/finall for dl in $(cat $dir/final); do { echo -n "Working... for $f " >&2 curl -w "%{url_effective}\n" -L -s -I -S "$dl" -o /dev/null } ;done | grep -v "HTTP" | awk '!seen[$0]++' >>$dir/final mkdir test cd test for f in $(sed 's/http:\/\/profile.gigaset.net\///g' $dir/final);do mkdir -p $f; echo -n "winx ";done tree -T "profile.gigaset.net" -C -H http://profile.gigaset.net -o $dir/tree.html mv $dir/final $dir/urllist git clone https://${GH_TOKEN}@github.com/danog/gigaclone.git $tmp/git cd $tmp/git cp $dir/urllist . git add -A git commit -m "Updated url list" git push origin master &>/dev/null git checkout gh-pages cp $dir/tree.html . git commit -m "Updated dir tree" git push origin gh-pages &>/dev/null cd $dir echo "Clean up." rm -fr $tmp