bench.py

#!/usr/bin/env python

"""
This is a little benchmarking script to exercise bagit.make_bag and
bagit.validate using 1-8 parallel processes. It will download some images 
from NASA for use in bagging the first time it is run.
"""

import ftplib
import os
import shutil
import timeit

import bagit

# fetch some images from NASA to bag up

if not os.path.isdir("bench-data"):
    print("fetching some images to bag up from nasa")
    os.mkdir("bench-data")
    ftp = ftplib.FTP("nssdcftp.gsfc.nasa.gov")
    ftp.login()

    ftp.cwd("/pub/misc/photo_gallery/hi-res/planetary/mars/")
    files = []
    ftp.retrlines("NLST", files.append)

    for file in files:
        print(("fetching %s" % file))
        fh = open(os.path.join("bench-data", file), "wb")
        ftp.retrbinary("RETR %s" % file, fh.write)
        fh.close()


# create bags using 1-8 processes

statement = """
import os
import bagit

if os.path.isdir('bench-data/data'):
    os.system("rm bench-data/bag*")
    os.system("mv bench-data/data/* bench-data/")
    os.system("rmdir bench-data/data")

bagit.make_bag('bench-data', processes=%s)
"""

for p in range(1, 9):
    t = timeit.Timer(statement % p)
    print(
        ("create w/ %s processes: %.2f seconds " % (p, (10 * t.timeit(number=10) / 10)))
    )


# validate a bag with 1-8 processes

shutil.copytree("bench-data", "bench-data-bag")
bagit.make_bag("bench-data-bag")

# validate bench-data using n processes
statement = """
import os
import bagit

bag = bagit.Bag('bench-data-bag')
bag.validate(processes=%s)
"""

# try 1-8 parallel processes
for p in range(1, 9):
    t = timeit.Timer(statement % p)
    print(
        (
            "validate w/ %s processes: %.2f seconds "
            % (p, (10 * t.timeit(number=10) / 10))
        )
    )

shutil.rmtree("bench-data-bag")