#!/bin/bash

#Author : Richard DEMONGEOT - http://www.demongeot.biz/outils/SiteMap.html

#Ce fichier est sous licence GPL V2.
#Je vous demanderai toutefois de bien vouloir me faire parvenir (par mail ou par
#un autre moyen à votre convenance) les modifications que vous êtes suceptibles
#d'y ajouter, ainsi que leurs justification.
#
#--------------------------------------------------------------------------------------
#
#This program is free software; you can redistribute it and/or modify
#it under the terms of the GNU General Public License as published by
#the Free Software Foundation; either version 2 of the License.
#
#Thanks to send me any change you can wrote to this project and their reasons.
#
#This program is distributed in the hope that it will be useful,
#but WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#GNU General Public License for more details.
#
#You should have received a copy of the GNU General Public License
#along with this program; if not, write to the Free Software
#Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

if [ $# -ne 1 ]
then
  exit 1
fi

#Variables

BASEURL=$1
CURRENTDIR=`pwd`
TMP=$CURRENTDIR"/tmp"
TEMP=$CURRENTDIR"/temp"
RESULTDIR=$CURRENTDIR"/result"
IGNORELIST="\.avi|\.mkv|\.pdf|#|\.jpg|\.png|\.tar"
#IGNORELIST="\.avi|\.mkv"
URLTODO=1

#Mise en place de l'environnement
if [ ! -d $TMP ]
then
  mkdir -p $TMP
fi

if [ ! -d $TEMP ]
then
  mkdir -p $TEMP
fi

if [ ! -d $RESULTDIR ]
then
  mkdir -p $RESULTDIR
fi

#Fonctions
#Fonction de suppression d'un fichier, avec virification de l'existance de celui-ci
delete ()
{
  i=1
  while [ $i -le $# ]
  do
    eval FILE=\$$i
    #eval sert a evaluer une commande. Ici, File va prendre pour valeur
    #le i-ieme argument de la fonction (le delete etant le 0)
    if [ -f $FILE ]
    then
      rm -f $FILE
    else
      echo Fichier $FILE introuvable
    fi
    i=`expr $i + 1`
  done
}

#Fonction pour eliminer les doublons dans fichier passe en argument
#Cette fonction necessite la fonction delete declare ci-dessus
ElimineDoublon ()
{
  sort $1 > /tmp/sort.tmp
  j=""
  for i in `cat /tmp/sort.tmp`
  do
    if [ "$i" != "$j" ] && [ "$j" != "" ]
    then
      echo $j >> /tmp/trt.tmp
    fi
    j=$i
  done
  echo $j >> /tmp/trt.tmp
  mv /tmp/trt.tmp $1
  delete /tmp/sort.tmp
}

#Fonction pour tester et supprimer les fichiers qui sont vides
#Cette fonction necessite la fonction delete declare ci-dessus
TestSize ()
{
  FileSize=`ls -l $1 | awk {'print $5'}`
  if [ 0 -eq $FileSize ]
  then
    delete $1
  fi
}

CrowlLink ()
{
  FOLDER1=`echo $1 | sed "s/^\///"`
  FOLDER=`echo $FOLDER1 | sed "s/\// /g"`
  NOMBRE=`echo $FOLDER | awk '{print NF}'`
  NOMBRE1=`expr $NOMBRE - 1`
  TESTFOLDER=`echo $1 | grep "/$"`
  if [ "" != "$TESTFOLDER" ]
  then
    NOMBRE=`expr $NOMBRE + 1`
  fi

  if [ $NOMBRE1 -ne 0 ]
  then
    REP=`echo $1 | cut -d '/' -f 2-$NOMBRE`
  else
    REP=""
  fi
 
  if [ ! -d $TEMP/$REP ]
  then
    mkdir -p $TEMP/$REP
  fi
  #REPERTOIRE=`echo $REP | sed "s/\//\\\\\\\\\\\//g"`
  #REPERTOIRE="/"$REPERTOIRE
  cd $TEMP/$REP
  wget -c $BASEURL/$1
  FILE=`ls -ltr $TEMP/$REP | tail -n 1 | awk '{print $8}'`
  grep "href=" $TEMP/$REP/$FILE | grep "<a" > $TMP"/AllLinks"
  sed "s/.*href=//g" $TMP"/AllLinks" > $TMP"/AllLinks1"
  sed "s/>.*//g" $TMP"/AllLinks1" > $TMP"/AllLinks2"
  egrep -v "http|ftp|mailto" $TMP"/AllLinks2" > $TMP"/LocalLinks"
  egrep -v "$IGNORELIST" $TMP"/LocalLinks" > $TMP"/LocalLinksToSee"
  cut -d '"' -s -f 2 $TMP"/LocalLinksToSee" >> $TMP"/LocalLinksShowing"
  cat $TMP"/LocalLinksShowing" | grep "^/" > $TMP"/GoodURL"
  cat $TMP"/LocalLinksShowing" | grep -v "^/" > $TMP"/BadURL"
  #cat $TMP"/BadURL"
  #cat $TMP"/GoodURL"
  #sed "s/^/$REPERTOIRE\//" $TMP"/BadURL"
  #sed "s/^/$REP\//" $TMP"/BadURL" >> $TMP"/GoodURL"
  for ligne in `cat $TMP"/BadURL"`
  do
    if [ "" != "$REP" ]
    then
      echo "/"$REP"/"$ligne >> $TMP"/GoodURL"
    else
      echo "/"$ligne >> $TMP"/GoodURL"
    fi
  done
  mv $TMP"/GoodURL" $TMP"/LocalLinksShowing"
  #cut -d "'" -s -f 2 $TMP"/LocalLinksToSee" >> $TMP"/LocalLinksShowing"
  echo $1 >> $TMP"/crowled"
  IGNORELIST=$IGNORELIST"|^$1\$"
  ElimineDoublon $TMP"/LocalLinksShowing"
}

#Main program
rm -rf $TMP/*
rm -rf $TEMP/*
echo "/" >> $TMP"/LocalLinksShowing"

while [ $URLTODO -ne 0 ]
do
  for URL in `cat $TMP"/LocalLinksShowing"`
  do
    EXIST=`grep ^$URL$ $TMP"/crowled"`
    if [ "$EXIST" = "" ]
    then
      CrowlLink $URL
    fi
    NBLIGNESHOWING=`cat $TMP"/LocalLinksShowing" | wc -l`
    NBLIGNECROWLED=`cat $TMP"/crowled" | wc -l`
    if [ $NBLIGNESHOWING -eq $NBLIGNECROWLED ]
    then
      URLTODO=0
    fi
  done
done

sort $TMP"/crowled" > $TMP"/crowledb"
echo "<urlset xsi:schemaLocation=\"http://www.google.com/schemas/sitemap/0.84 http://www.google.com/schemas/sitemap/0.84/sitemap.xsd\">" > $RESULTDIR"/sitmap.xml"
for ligne in `cat $TMP"/crowledb" | grep "^/"`
do
  echo "  <url><loc>$BASEURL$ligne</loc></url>" >> $RESULTDIR"/sitmap.xml"
done

echo "</urlset>" >> $RESULTDIR"/sitmap.xml"
