#!/bin/bash
# Script to automatically Generate an URL List for use with Yahoo!
# Output file is also suitable for use with Google Sitemap Generator and HTML_sitemap_gen
# Copyright Ben Tasker 2010
# Released under the GNU GPL
# See the LICENSE file or http://benscomputer.no-ip.org/LICENSE3 for more details

#############################################################
#							    #
#							    #
#                    URLList Gen V1.5                       #
#							    #
#							    #
#############################################################

# Set your variables here

# Set the address of the website you wish to crawl, use a lowercase http

WEBADDRESS="http://benscomputer.no-ip.org"

# Where do you want the script to put the urllist?
URLLISTLOCAT="/mnt/.apacheroot/www/localhost/htdocs/urllist.txt"

# Where do you want the Temporary files to be stored? (Wget doesn't like /tmp on my system for some reason)
TMPLOCAT="/home/ben"


# Code starts

# Check for existence of a lockfile

if [ -e /tmp/URL_List_gen.sh ]
then

echo "Lockfile Detected - Exiting"
exit

fi

# Create lockfile

echo "" > /tmp/URL_List_gen.sh
chmod 777 /tmp/URL_List_gen.sh

# Prepare the environment and make sure things are cleaned up

rm urllist

# Crawl the site and find all URLS that are available
wget -l 0 -U "Url_list_gen" -r --delete-after -o urllist $WEBADDRESS

# The output is currently packed with stuff we don't need, lets grep it out

cat urllist | grep "http" > urllist2
cat urllist2 | grep -v "Location" > urllist
cat urllist | grep -v "response" > urllist2
cat urllist2

# this will give a file full of lines in the format
# --TIMESTAMP -- http://address/page
# So we need to get rid of the timestamp and any excess spaces
# Would be nice if we could do it character by character but people do use - in web addresses (numbers too)

echo "" > urllist
for i in $( cat urllist2 )
do

# This command seems to identify the double space between the timestamp and the address as a new record
# So lets take advantage and seperate them onto different lines
echo $i >> urllist

done

# Now lets get rid of the timestamps
cat urllist | grep "http" > urllist2

# The list contains any page on our site that we have linked to, but if a link is broken then we don't really want
# to make things worse by putting it into sitemaps, lets run wget on each one and check the status
rm urllist
for i in $( cat urllist2 )
do
wget -U "Url_list_gen" --spider $i
if [ "$?" == "0" ]
then
# Wget reported no errors, should mean a HTTP 200 Status
echo "$i" >> urllist
fi
done

# Finally we just need to remove a couple of things we don't really want Indexed by Yahoo etc.
cat urllist | grep -iv "robots.txt" > urllist2
cat urllist2 | grep -iv "[.]ico" > urllist

# Tidy up
rm urllist2

#Move the URLLIST to its final location
mv urllist $URLLISTLOCAT

# Remove Lockfile as we are done
rm -f /tmp/URL_List_gen.sh 2> /dev/null

exit