#!/bin/sh
#---------------------------------------------------------------
#    unRAIDFindDuplicates.sh
#    ~~~~~~~~~~~~~~~~~~
# A utility to find duplicate files in unRAID shares.
# Strictly speaking we are looking for files with identical
# names in more than one location, rather than looking for
# files with duplicate contentns (but differing names). The
# check for duplicate files can also check if the contents
# are identical and give appropriate warnings if differences 
# are found
#
# The basic functionality has been extended slightly to include
# related functionality that has proved useful such as checking
# for empty directories; zero length files or checking array 
# contents against backup disks.
#
# CHANGE HISTORY
# ~~~~~~~~~~~~~~
# Version 1.0  09 Sep 2014  First Version  
# Version 1.1  10 Sep 2014  Got the -b option working to identify files where the name and
#                           size are the same but the contents differ.
#                           Added -q option to suppress all console output while running.  Useful
#                           if you want to run the script as a cron job or similar.
#                           Added warning if file sizes differ
# Version 1.2  13 Sep 2014  Added the -D option to check extra disk
# Version 1.3  01 Oct 2014  Added -f and -F options to list empty (duplicated) directories.
#                           Added -z and -Z options to list zero length (duplicated) files.
#                           Fix:  Allow for shares that have spaces in their names.
# Version 1.4   07 Oct 2014 Fix:  Use process ID in /tmp filenames to allow multiple copies of
#                           the script to be run in parallel without interfering with each other.
# Version 1.5   07 Mar 2016 Fix: Incorrect reporting of file size mismatches when sparse
#                           files involved.
# Version 1.?   ?? ??? ???? (planned - development in progress)
#                           Added -x option to only report files that are different
#                           Added -X option to report files that are different to a backup disk
# 
# Dave walker (a.k.a. itimpi)
#---------------------------------------------------------------

version="1.5"

start_directory=`pwd`
progname=$0
program_dir=`dirname $0`
if [ "preogram_dir" = "." ]
then
  program_dir=$start_directory
fi
arg_count=$#
options=$*

usage() {
  echo
  echo "Usage: $progname [-v] ]-q] [-b ] [-c] -d exclude_disk] [-o filename] [-i dirname] [-e dirname] [-f/F] [z|Z]"
  echo
  echo " -b      =   If duplicate names found, do a binary compare of the files as well"
  echo "             If omitted, then only a compare on file name is done"
  echo "             NOTE.  Using this option slows things down A LOT as it needs to"
  echo "                    read every byte of files whose names match to compare them"
  echo " -c      =   Ignore case when checking names."
  echo "             This can help with the fact that linux is case sensitive on filenames,"
  echo "             whereas Windows, and by implication Samba, is case independent. This"
  echo "             can lead to unexpected filename collisions."
  echo " -d exclude_disk  (may be repeated as many times as desired)"
  echo "             The default behavior is to include all disks in the checks."
  echo "             Use this to exclude a specific disk from being included in the checks"
  echo " -D path     Treat the given path as if it was an array disk"
  echo "             (may be repeated as many times as necessary)."
  echo "             Can be useful to test if files on an extra disk already exist in the array."
  echo " -e exclude_dir  (may be repeated as many times as desired)"
  echo "             Use this to exclude a particular share/directory from being included in the checks"
  echo " -f          List any empty folders (directories) that are duplicates of non-empty folder on"
  echo "             another disk. These can be left behind when you remove duplicate files, but notg"
  echo "             their containing folder.  However empty folders are also possible in normal"
  echo "             operation so finding these is not necessarily an issue."
  echo " -F          List any empty folders even if they are not duplicated on another drive.  This"
  echo "             may be perfectly valid but at least this helps you decide if this is so."
  echo " -i include_dir  (may be repeated as many times as desired)"
  echo "             Use this to include a particular share/directory to be included in the checks"
  echo "             If omitted, then all top level folders on each disk (except for those"
  echo "             specifically excluded via the -e option(s)) will be included in the checks"
  echo " -o filename Specify the output filename to which a copy of the results should be sent."
  echo "             If omtted then the results are sent to the file duplicates.txt on the root"
  echo "             of the flash drive e.g. /boot/duplicates.txt from linux"
  echo " -q      =   Quiet mode."
  echo "             No console output while running.  Need to see results file for output."
  echo " -v      =   verbose mode.  Additional Details are produced as progress proceeds"
  echo " -V      =   print program version "
  echo " -x      =   Only report file mismatches on time/size (default) or content (if -b also used)"
  echo "             Does not simply report the fact that there is a duplicate if they appear identical."
  echo " -X path =   Check the array against the given disk and report if files on the array are"
  echo "             either missing or appear to be different size.  Use the -b option as well if you"
  echo "             want the file contents checked as well.  Useful for checking whether you have files"
  echo "             on a backup disk that are not also on the main array.  It is assumed that the path"
  echo "             specified conatins files in the same folder structure as is present on the array."
  echo "  z          Report zero length files that are also duplicates.   These are not necessarily"
  echo "             wrong, but could be a remnant of some previous issue or failed copy"
  echo "  Z          Report xero length files even when they are not duplicates."
  echo ""
  echo "EXAMPLES:"
  echo
  echo "To check all shares on all disks except disk 9"
  echo "  $progname -d 9"
  echo
  echo "To check just the TV share"
  echo "  $progname -i TV"
  echo 
  echo "To check all shares except the Archives share"
  echo "  $progname  -e Archives"
  echo
  echo "TIP:  This program runs much faster if all drives are spun up first"
  exit 0
}

# ----------- Support functions to help with reporting ----------------

#  Output message to console in verbose mode

verbose_to_console() {
  if [ ! $verbose = "no" ] ; then
    to_console "$*"
  fi
}

#  Output message to console and results file in verbose mode

verbose_to_both() {
  if [ ! $verbose = "no" ] ; then
    to_both "$*"
  fi
}

#  Output message taking into consideration quiet mode

to_console() {
  if [ $quiet_mode = "no" ] ; then
    echo -E "$*"
  fi
}

to_both() {
  if [ $quiet_mode = "no" ] ; then
    echo -E "$*" | tee -a $results_file
  else
    echo -E "$*" >> $results_file
  fi
}

# Geta list of drives for share file/folder
#  $1  path to cech
#  $2  filename for results
#  $3  set to "-d" for directories 
get_drive_list() {
  echo -n "" > $2
  ls -1 $3 /mnt/disk*/"$1" | \
  while read xx; do
    echo "$xx" | cut -f 3 --delimiter=/ >> $2
  done
  # Now allow for cache drive
  if [ "$3" = "" ] ; then
    if [ -f "/mnt/cache/$1" ] ; then
    echo "cache" >> $2
    fi
  elif [ -d /mnt/cache ] ; then
    if [ -d "/mnt/cache/$1" ] ; then
      echo "cache" >> $2
    fi
  fi  
}

#--------------------- Main Processing Logic ------------------------

# Check array is started

ls /mnt/disk* 1>/dev/null 2>/dev/null
if [ $? -ne 0 ] ; then
  echo "ERROR:  Array is not started"
  echo "        It needs to be running to perform this command"
  exit
fi

# Initialise program variables

verbose="no"
quiet_mode="no"
results_file="/boot/duplicates.txt"
binary_compare="no"
report_differences_only="no"
report_against_backup=""
sort_options=""
uniq_options=""
exclude_disks_count=0
include_disks_count=0
exclude_shares_count=0
include_shares_count=0
include_disks=
include_shares=
empty_directories="no"                  # Look for empty directories
empty_directories_duplicated="yes"      # .. report only if also duplicatged
zero_files="no"        # Look for zero length files
zero_files_duplicated="yes"             # ... report only if also duplicated
mismatches_only="no"
backup_path=""

# Parse options

while getopts "bcD:d:e:fFhi:o:qVvxX:zZ" opt; do
  case $opt in
    b ) binary_compare="yes" 
        ;;
    C ) sort_options="-f"
        uniq_options="-i"
        ;;
    D ) include_disks[$include_disks_count]="/mnt/disk${OPTARG}"
        include_disks_count=$((include_disks_count+1)) 
      ;;
    d ) if [ ! -d "/mnt/disk${OPTARG}" ]
        then
          echo "ERROR: /mnt/disk${OPTARG} not present"
          echo "TIP: You must not include /mnt/disk as this is added autoamtically"
          exit 2
        fi
        exclude_disks[$exclude_disks_count]="/mnt/disk${OPTARG}"
        exclude_disks_count=$(($exclude_disks_count+1)) 
        ;;
    e ) exclude_shares[$exclude_shares_count]="${OPTARG}"
        exclude_shares_count=$(($exclude_shares_count+1)) 
        ;;
    f ) empty_directories="yes"
        ;;
    F)  empty_directories_duplicated="no"
        empty_directories="yes"
        ;;
    h|"-help" ) usage
        ;;
    i ) if [ ! -d "/mnt/user/${OPTARG}" ]
        then
          echo "ERROR:  share /mnt/user/$OPTARG} not found"
          echo "TIP: You must not include /mnt/user as this is added autoamtically"
          exit 2
        fi
      include_shares[$include_shares_count]="${OPTARG}"
        include_shares_count=$(($include_shares_count+1)) 
        ;;
    o ) results_file="${OPTARG}"
        ;;
    q ) quiet_mode="yes"
        ;;
    v ) verbose="yes" 
        ;;
    V ) echo $progname "version "$version
        exit 0 
        ;;
    x ) echo "ERROR: Option not yet ready for use"
        exit -2
        mismatches_only="yes"
        ;;
    X ) echo "ERROR:  Option not yet ready for use"
        exit -2
        if [ ! -d "/mnt/disk${OPTARG}" ]
        then
          echo "ERROR: disk${OPTARG} not present"
          exit 2
        fi
        backup_path="/mnt/disk${OPTARG}"
        mismatches_only="yes "
        ;;
    z ) zero_files="yes"
        ;;
    Z)  zero_files_duplicated="no"
        zero_files="yes"
        ;;
    \?) usage >&2 ; exit 
        ;;
    * ) usage
        ;;
  esac
done

# Work out list of disks to use

for d in `ls -d /mnt/disk* | sort -V`
do
  exclude="no"
  done_disks_count=0
  while test $done_disks_count -lt $exclude_disks_count
  do
    if [ $d = ${exclude_disks[$done_disks_count]} ]
    then
      exclude="yes"
    fi
    done_disks_count=$(($done_disks_count+1))
  done
  if [ $exclude = "no" ]
  then
    include_disks[$include_disks_count]="$d"
    include_disks_count=$(($include_disks_count+1))
  fi
done
# Add cache disk if present
if [ -d /mnt/cache ]
then
  include_disks[$include_disks_count]="/mnt/cache/"
  include_disks_count=$(($include_disks_count+1))
fi  

# Work out list of shares to use

if [ $include_shares_count -eq 0 ] ; then
  if [ -d /mnt/user0 ] ; then
    # Cache disk present - only look on main array for share folder names
    cd /mnt/user0
  elif [ -d /mnt/user ] ; then
    # cache disk not present - then all folders under /mnt/user are to be used
    cd /mnt/user
  else
    echo "ERROR:  User shares appear to not be used?"
    echo "        If you would like the script to handle this case"
    echo "        then please request that this feature be added."
    exit -2
  fi

  ls -d * > /tmp/$$_shares
  sort -V /tmp/$$_shares > /tmp/$$_shares_sorted
  while read s
  do
    exclude="no"
    done_shares_count=0
    while test $done_shares_count -lt $exclude_shares_count
    do
      if [ "$s" = "${exclude_shares[$done_shares_count]}" ]
      then
        exclude="yes"
      fi
      done_shares_count=$(($done_shares_count+1))
    done
    if [ $exclude = "no" ]
    then
      include_shares[$include_shares_count]="$s"
      include_shares_count=$(($include_shares_count+1))
    fi
  done < /tmp/$$_shares_sorted
fi

echo "COMMAND USED: " $progname $* > $results_file
echo >> $results_file

if [ ! $verbose = "no" ]; then
  verbose_to_console "" 
  verbose_to_console "============= STARTING unRAIDFIndDuplicates.sh ==================="
  verbose_to_console "" 
  verbose_to_both "Included disks:"
  done_disks_count=0
 
  while test $done_disks_count -lt $include_disks_count
  do
    verbose_to_both "   ${include_disks[$done_disks_count]}"
    done_disks_count=$(($done_disks_count+1))
  done
  verbose_to_both ""

  verbose_to_both "Included shares:"
  done_shares_count=0
  while test $done_shares_count -lt $include_shares_count
  do
    verbose_to_both "   ${include_shares[$done_shares_count]}"
    done_shares_count=$(($done_shares_count+1))
  done
  verbose_to_both ""

  verbose_to_both "List duplicate files"
  verbose_to_bpth ""
  
  if [ "$empty_directories" = "yes" ] ; then
    x="List empty Directories" 
    if [ "$empty_directories_duplicated" = "yes" ] ; then
      x="$x [only when duplicated]"
    else
      x="$x [even when not duplicated]"
    fi
    verbose_to_both $x
    verbose_to_both ""
  fi
  if [ "$zero_files" = "yes" ] ; then
    x="List zero-length files" 
    if [ "$zero_files_duplicated" = "yes" ] ; then
      x="$x [only when duplicated]"
    else
      x="x$ [even when not duplicated]"
    fi
    verbose_to_both $x
    verbose_to_both ""
  fi
fi

#  PHASE 1 - Create file lists
#            (and also empty directories list uf required)

done_disks_count=0
while test $done_disks_count -lt $include_disks_count
do
  touch /tmp/$$.$done_disks_count 
  cd ${include_disks[$done_disks_count]}
  verbose_to_both "checking ${include_disks[$done_disks_count]}"
  done_shares_count=0

  while test $done_shares_count -lt $include_shares_count
  do
    if [ -d "${include_shares[$done_shares_count]}" ]
    then
      verbose_to_both "    ${include_shares[$done_shares_count]}"
      find "${include_shares[$done_shares_count]}" -type f -print >> /tmp/$$.$done_disks_count
      if [ "$empty_directories" = "yes" ]
      then
        find "${include_disks[$done_disks_count]}/${include_shares[$done_shares_count]}" -type d -empty -print >> /tmp/$$_empty
      fi
      if [ "$zero_files" = "yes" ]
      then
        find "${include_disks[$done_disks_count]}/${include_shares[$done_shares_count]}" -type f -empty -print >> /tmp/$$_zero
      fi
    fi
    done_shares_count=$(($done_shares_count+1))
  done
  done_disks_count=$(($done_disks_count+1))
done

#  PHASE 2 - Sort results

# verbose_to_console "Sorting filelists"
sort $sort_options /tmp/$$.[0-9]* > /tmp/$$_sorted


#  PHASE 3 - Identify duplicates

# verbose_to_console "identifying duplicates"

cat /tmp/$$_sorted | uniq -d $uniq_options > /tmp/$$_duplicates

to_both "Duplicate Files"
to_both "---------------"
to_both ""

if [ -s /tmp/$$_duplicates ] ; then
  while read f
  do
    # Files on array drives
    ls /mnt/disk*"/$f" > /tmp/$$_matches
    # Allow for file also on cache drive
    if [ -f /mnt/cache"/$f" ] ; then
      echo /mnt/cache"/$f" >>/tmp/$$_matches 2>&1
    fi
    previous_file=""
    files_different=""
    # work through matches
    while read ff 
    do
      if [ $verbose = "no" ] ; then
        if [ "$previous_file" = "" ] ; then
          to_both "$f"
        fi
      else
        s=`ls -l "$ff"`
        verbose_to_both "$s"
        if [ -f /tmp/$$_difference ] ; then
          verbose_to_both `cat /tmp/$$_difference`
        fi
      fi

      # Do some consistency checks on files matching
      
      # Start with file size check
      ff_size=`du -b "$ff" | cut --field=1`
      if [ ! "$previous_file" = "" ] ; then
        if [ $previous_file_size -ne $ff_size ] ; then
           to_both "  WARNING: File sizes are different"
        else
          # Size matches - should we do a Binary file compare?
          if [ $binary_compare = "yes" ] ; then
#            verbose_to_console "comparing $previous_file"
#            verbose_to_console "       to $ff"
            cmp "$previous_file" "$ff" >/dev/null 2>&1
            if [ $? -ne 0 ] ; then
              to_both "  WARNING: File contents are different"
            else
              verbose_to_console "  Files the same"
            fi
          else
            to_both ""          
          fi
        fi
      fi
      previous_file="$ff"
      previous_file_size=$ff_size
    done </tmp/$$_matches
  done </tmp/$$_duplicates
else
  to_both "** No duplicate files found **"
fi

#  see about empty directories

if [ "$empty_directories" = "yes" ] ; then
  verbose_to_both ""
  verbose_to_both "Empty Directories"
  verbose_to_both "-----------------"
  if [ -s /tmp/$$_empty ] ; then
    empty_directories_listed="no"
    while read f ; do
      # Remove the /mnt/disk* from start of strings for listing purposes
      ff=`echo $f | cut --delimiter=/  --fields=4-`
      fd=`echo $f | cut --delimiter=/  --fields=3`
      get_drive_list "$ff" $$.tmp "-d"
      if [ "$empty_directories_duplicated" = "yes" ] ; then
        # Check to see if there is also a additonal drives
        if [ `cat $$.tmp|wc -l` -gt 1 ] ; then
          verbose_to_both "  $ff"
          verbose_to_both "    $fd [empty]" 
          empty_directories_listed="yes"    
          while read x ; do  #  
            if [ ! $x = $fd ] ; then
              verbose_to_both "    $x"
            fi
          done < $$.tmp
        fi
      else
        verbose_to_both "  $ff"
        verbose_to_both "    $fd" 
        empty_directories_listed="yes"      
      fi
    done </tmp/$$_empty 
    if [ "$empty_directories_listed" = "no" ] ; then
      verbose_to_both "** No empty directories found **"
    fi
  else
    verbose_to_both "** No empty directories Found **"
  fi
fi


#  see about zero length files

if [ "$zero_files" = "yes" ] ; then
  verbose_to_both ""
  verbose_to_both "Zero Length Files"
  verbose_to_both "-----------------"
  if [ -s /tmp/$$_zero ] ; then
    zero_files_listed="no"
    while read f
    do
      # Remove the /mnt/disk*/ from start of strings for listing purposes
      ff=`echo "$f" | cut -s --delimiter=/  --fields=4-` 
      fd=`echo "$f" | cut --fields=3 --delimiter=/ `
      get_drive_list "$ff" $$.tmp
      if [ "$zero_files_duplicated" = "yes" ] ; then
        # Check to see if there is also a additonal drives
        if [ `cat $$.tmp|wc -l` -gt 1 ] ; then
          verbose_to_both "  $ff"
          verbose_to_both "    $fd [zero length]" 
          zero_files_listed="yes"    
          while read x ; do
            if [ ! $x = $fd ] ; then
              verbose_to_both "    $x"
            fi
          done < $$.tmp
        fi
      else
        verbose_to_both "  $ff"
        verbose_to_both "    $fd" 
        zero_files_listed="yes"      
      fi
    done </tmp/$$_zero 
    if [ "$zero_files_listed" = "no" ] ; then
      verbose_to_both "** No zero length" \("unduplicate"\) "files Found **"
    fi
  else
    verbose_to_both "** No zero length files Found **"
  fi
fi

# TIDY UP

cd $start_directory

rm -f /tmp/$$*

verbose_to_console "" 
verbose_to_console "============= COMPLETED unRAIDFIndDuplicates.sh ==================="