pcompare

#!/usr/bin/env python
# Copyright Genome Research Ltd 2014
# Author Guy Coates <gmpc@sanger.ac.uk>

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.

"""
Parallel directory comparision. This program compares the contents of
two directory trees.

"""
from mpi4py import MPI
from lib.parallelwalk  import ParallelWalk
import argparse
import os
import stat
import sys
import traceback
import hashlib
import ctypes
from lib.readdir import readdir
import math

def safestat(filename):
    """lstat sometimes get Interrupted system calls; wrap it up so we can
    retry"""
    while True:
        try:
            statdata = os.lstat(filename)
            return(statdata)
        except IOError, error:
            if error.errno != 4:
                raise

def mungePath(src, dst, f):
    """Convert the sourcepath to the desinationpath"""
    suffix = f.partition(src)[2]
    dest = dst + suffix
    return(dest)


class pcmp(ParallelWalk):
    """Extend the generic parallel walk to stat and compare files
    """
    def ProcessFile(self, filename):
        """
        stat the source and destination files; complain if they are different.
        """
        dstfile = mungePath(dir1, dir2, filename)
        try:
            srcsize = safestat(filename).st_size
        except OSError as error:
            print "cannot access `%s':" % filename,
            print os.strerror(error.errno)
            srcsize = None
        try:
            dstsize = safestat(dstfile).st_size
        except OSError as error:
            print "cannot access `%s':" % dstfile,
            print os.strerror(error.errno)
            dstsize = None

        self.results += 1
        if srcsize <> dstsize:
            print "R%i: MISMATCH: %s %s: %s %s" %(rank, filename, dstfile, srcsize, dstsize)

        if VERBOSE:
            print "R%i: %s %s: %s %s" %(rank, filename, dstfile, srcsize, dstsize)


class MPIargparse(argparse.ArgumentParser):
    """Subclass argparse so we can add a call to Abort, to tidy up MPI bits and pieces."""
    def error(self,message):
        self.print_usage(sys.stderr)
        Abort()

    def print_help(self, file=None):
        argparse.ArgumentParser.print_help(self, file=None)
        Abort()

def Abort():
    print ""
    MPI.COMM_WORLD.Abort(0)
    exit (1)

def parseargs():
    parser = MPIargparse(
        formatter_class = argparse.RawDescriptionHelpFormatter,
        description = """Parallel directory compare. This program will walk 
two directory trees and compare the sizes all files. It should be invoked
via mpirun."""

)
    parser.add_argument("DIR", nargs=2, help="Directories to compare.")
    parser.add_argument("-v", help="verbose", default=False, action="store_true")

    if len(sys.argv) == 1:
        parser.print_help()
        Abort()

    args = parser.parse_args()
    return(args)

# Begin main program. We have to catch exceptions
# and call MPIABORT, otherwise other ranks will
# hang indefinately.
try:
    comm = MPI.COMM_WORLD
    rank = comm.Get_rank()
    workers = comm.size

    args = parseargs()
    dir1 = args.DIR[0].rstrip("/")
    dir2 = args.DIR[1].rstrip("/")
    VERBOSE = args.v
    
    crawler = pcmp(comm, results=0)
    results = crawler.Execute(dir1)
    if rank == 0:
        total = sum(results)
        print "%i files compared" %total
    exit(0)

except (Exception, KeyboardInterrupt), err:
    print "Exception on rank %i" %rank
    print traceback.format_exc()
    Abort()