#! /usr/bin/perl
use warnings;
use strict;
use threads;
use threads::shared;

use File::Find ();
use File::Compare ();

use vars qw(@RootDirs %Files);

@RootDirs = ('H:\test\');

print STDERR "Reading Files and sorting them...\n";
my $filesCount = 0;
File::Find::find(sub {
    if (-f $_) {
    push (@{ $Files{-s _} }, $File::Find::name);
    if ($filesCount and $filesCount % 5000 == 0) {
        print STDERR "$filesCount files read\n";
    } # if
    $filesCount++;
    } # if
}, @RootDirs);
print STDERR "$filesCount files read and sorted\n\n";

my $compareCount : shared = 0;
my $doubleCount : shared = 0;
my $finished : shared = 0;

print STDERR "Comparing files...\n";
my $thread = threads->create(\&CompareFiles);
while ($finished == 0) {
    sleep(5);
    printf STDERR "%2d%%\t$compareCount/$filesCount => $doubleCount\n", $compareCount*100/$filesCount;
} # while

$thread->join();
# ------------------------------------------------------------
sub CompareFiles {

    foreach (keys %Files) {
    my @items = @{ $Files{$_} };
    my $count = $#items;
    $compareCount += $count+1;

    next if $count == 0;
    for my $i (0..$#items-1) {
        for my $j ($i+1..$#items) {
        my $cmp = File::Compare::compare($items[$i], $items[$j]);
        if ($cmp == 0) {
            $doubleCount++;
            print "$doubleCount\tA:\t$items[$i]\n";
            print "$doubleCount\tB:\t$items[$j]\n\n";
        } # if
        } # for
    } # for
    } # foreach

    print STDERR "Compared: $compareCount files: 2*$doubleCount Duplicates\n";

    $finished = 1;
} # CompareFiles
# ------------------------------------------------------------

#!/usr/bin/perl
# finddupes - find duplicate files by MD5 digest
#
use strict;
use warnings;
use File::Find;
use Digest::MD5;

my %FilesBySize;

# default: current directory
@ARGV = qw/./ unless @ARGV;

# find all files
find ({wanted => \&add_file, no_chdir => 1}, @ARGV);

find_duplicates();

exit 0;

###########################################

sub add_file {
    return unless -f $_;
    my $size = -s _;
    return unless $size;
    push @{$FilesBySize{$size}}, $File::Find::name;
}

sub find_duplicates {
    for my $s (sort {$a <=> $b} keys %FilesBySize) {
        next unless @{$FilesBySize{$s}} > 1; # more than 1 file?

        my %md5s = ();
        for my $f (@{$FilesBySize{$s}}) {
            my $md5 = calc_md5 ($f);
            push @{$md5s{$md5}}, $f;
        }
        for my $md5 (keys %md5s) {
            if (@{$md5s{$md5}} > 1) {
                print "Identische MD5-Summe (mit $s Bytes): \n";
                print "\t$_\n" for @{$md5s{$md5}};
            }
        }
    }
}

sub calc_md5 {
    my $file = shift;
    open(FILE, $file) or warn "can't open '$file': $!\n";
    binmode(FILE);
    my $md5 = Digest::MD5->new;
    while (<FILE>) {
        $md5->add($_);
    }
    close(FILE);
    return $md5->b64digest;
}

http://www.cits.rub.de/imperia/md/content/magnus/letter_of_rec.ps
http://www.cits.rub.de/imperia/md/content/magnus/order.ps