diff -r ac90b03614ea autobuild/autobuild_tool_edit.py --- a/autobuild/autobuild_tool_edit.py Mon Aug 22 13:28:27 2011 -0700 +++ b/autobuild/autobuild_tool_edit.py Fri Oct 14 11:29:52 2011 -0500 @@ -84,6 +84,7 @@ self.arguments except AttributeError: self.arguments = { + 'archive': Archive, 'configure': Configure, 'build': Build, 'package': Package, @@ -255,6 +256,56 @@ self.config.package_description.platforms.pop(name) +class Archive(InteractiveCommand): + + ARGUMENTS = ['format', 'hash_algorithm', 'platform'] + + ARG_DICT = { 'format': {'help':'Archive format (e.g zip or tbz2)'}, + 'hash_algorithm': {'help':'The algorithm for computing the archive hash (e.g. md5)'}, + 'platform': {'help':'The name of the platform archive to be configured'} + } + + HELP = "Platform-specific archive configuration" + + def __init__(self, config): + stream = StringIO() + stream.write("Current platform archive settings:\n") + archives = {} + for platform, description in config.get_all_platforms().iteritems(): + if description.archive is not None: + archives[platform] = description.archive + configfile.pretty_print(archives, stream) + self.description = stream.getvalue() + stream.close() + self.config = config + + def _create_or_update_platform_archive(self, platform, format, hash_algorithm): + try: + platform_description = self.config.get_platform(platform) + except configfile.ConfigurationError: + platform_description = configfile.PlatformDescription({'name': platform}) + self.config.package_description.platforms[platform] = platform_description + if platform_description.archive is None: + platform_description.archive = configfile.ArchiveDescription() + platform_description.archive.format = format + platform_description.archive.hash_algorithm = hash_algorithm + + def run(self, platform=get_current_platform(), format=None, hash_algorithm=None): + """ + Configure platform archive details. + """ + self._create_or_update_platform_archive(platform, format, hash_algorithm) + + def delete(self, platform=get_current_platform(), **kwargs): + """ + Delete the named config value. + """ + print "Deleting %s archive entry." % platform + platform_description = self.config.get_platform(platform) + if platform_description is not None: + platform_description.archive = None + + class _package(InteractiveCommand): def __init__(self, config): diff -r ac90b03614ea autobuild/autobuild_tool_package.py --- a/autobuild/autobuild_tool_package.py Mon Aug 22 13:28:27 2011 -0700 +++ b/autobuild/autobuild_tool_package.py Fri Oct 14 11:29:52 2011 -0500 @@ -27,7 +27,7 @@ The package sub-command works by locating all of the files in the build output directory that match the manifest specified in the configuration file. The manifest can include platform-specific and -platform-common files, and they can use glob-style wildcards. +platform-common files which may use glob-style wildcards. The package command optionally enforces the restriction that a license string and a valid licensefile for the package has been specified. The operation @@ -43,6 +43,7 @@ * license_file (assumes LICENSES/.txt otherwise) """ +import hashlib import sys import os import tarfile @@ -54,6 +55,7 @@ import autobuild_base from connection import SCPConnection, S3Connection from common import AutobuildError +from zipfile import ZipFile, ZIP_DEFLATED logger = logging.getLogger('autobuild.package') @@ -87,17 +89,22 @@ default=True, dest='check_license', help="do not perform the license check") + parser.add_argument( + '--archive-format', + default=None, + dest='archive_format', + help='the format of the archive (tbz2 or zip)') def run(self, args): config = configfile.ConfigurationDescription(args.autobuild_filename) - package(config, args.platform, args.archive_filename, args.check_license, args.dry_run) + package(config, args.platform, args.archive_filename, args.archive_format, args.check_license, args.dry_run) class PackageError(AutobuildError): pass -def package(config, platform_name, archive_filename=None, check_license=True, dry_run=False): +def package(config, platform_name, archive_filename=None, archive_format=None, check_license=True, dry_run=False): """ Create an archive for the given platform. """ @@ -138,7 +145,23 @@ for f in files: logger.info('added ' + f) else: - _create_tarfile(tarfilename, build_directory, files) + archive_description = platform_description.archive + format = _determine_archive_format(archive_format, archive_description) + if format == 'tbz2': + _create_tarfile(tarfilename + '.tar.bz2', build_directory, files) + elif format == 'zip': + _create_zip_archive(tarfilename + '.zip', build_directory, files) + else: + raise PackageError("archive format %s is not supported" % format) + + +def _determine_archive_format(archive_format_argument, archive_description): + if archive_format_argument is not None: + return archive_format_argument + elif archive_description is None or archive_description.format is None: + return 'tbz2' + else: + return archive_description.format def _generate_archive_name(package_description, platform_name, suffix=''): @@ -150,7 +173,6 @@ name = package_name + '-' + package_description.version + '-' name += platform_name + '-' name += time.strftime("%Y%m%d") + suffix - name += '.tar.bz2' return name @@ -203,8 +225,45 @@ # Not using logging, since this output should be produced unconditionally on stdout # Downstream build tools utilize this output print "wrote %s" % tarfilename - import hashlib - fp = open(tarfilename, 'rb') + _print_hash(tarfilename) + + +def _create_zip_archive(archive_filename, build_directory, file_list): + if not os.path.exists(os.path.dirname(archive_filename)): + os.makedirs(os.path.dirname(archive_filename)) + current_directory = os.getcwd() + os.chdir(build_directory) + try: + archive = ZipFile(archive_filename, 'w', ZIP_DEFLATED) + added_files = set() + for file in file_list: + _add_file_to_zip_archive(archive, file, archive_filename, added_files) + archive.close() + finally: + os.chdir(current_directory) + print "wrote %s" % archive_filename + _print_hash(archive_filename) + + +def _add_file_to_zip_archive(zip_file, unormalized_file, archive_filename, added_files): + file = os.path.normpath(unormalized_file) + if file not in added_files: + added_files.add(file) + else: + logger.info('skipped duplicate ' + file) + return + try: + zip_file.write(file) + logger.info('added ' + file) + if os.path.isdir(file): + for f in os.listdir(file): + _add_file_to_zip_archive(zip_file, os.path.join(file, f), archive_filename, added_files) + except: + raise PackageError("unable to add %s to %s" % (file, archive_filename)) + + +def _print_hash(filename): + fp = open(filename, 'rb') m = hashlib.md5() while True: d = fp.read(65536) @@ -212,4 +271,5 @@ m.update(d) # Not using logging, since this output should be produced unconditionally on stdout # Downstream build tools utilize this output - print "md5 %s" % m.hexdigest() + print "md5 %s" % m.hexdigest() + diff -r ac90b03614ea autobuild/common.py --- a/autobuild/common.py Mon Aug 22 13:28:27 2011 -0700 +++ b/autobuild/common.py Fri Oct 14 11:29:52 2011 -0500 @@ -43,6 +43,7 @@ import shutil import subprocess import tarfile +from zipfile import ZipFile, is_zipfile import tempfile import urllib2 @@ -270,18 +271,13 @@ if not os.path.exists(cachename): logger.error("cannot extract non-existing package: %s" % cachename) return False - - # Attempt to extract the package from the install cache - logger.debug("extracting from %s" % cachename) - tar = tarfile.open(cachename, 'r') - try: - # try to call extractall in python 2.5. Phoenix 2008-01-28 - tar.extractall(path=install_dir) - except AttributeError: - # or fallback on pre-python 2.5 behavior - __extractall(tar, path=install_dir) - - return tar.getnames() + if tarfile.is_tarfile(cachename): + return __extract_tar_file(cachename, install_dir) + elif is_zipfile(cachename): + return __extract_zip_archive(cachename, install_dir) + else: + logger.error("package %s is not archived in a supported format" % cachename) + return False def remove_package(package): """ @@ -382,6 +378,29 @@ # ###################################################################### +def __extract_tar_file(cachename, install_dir): + + # Attempt to extract the package from the install cache + logger.debug("extracting from %s" % cachename) + tar = tarfile.open(cachename, 'r') + try: + # try to call extractall in python 2.5. Phoenix 2008-01-28 + tar.extractall(path=install_dir) + except AttributeError: + # or fallback on pre-python 2.5 behavior + __extractall(tar, path=install_dir) + + return tar.getnames() + +def __extract_zip_archive(cachename, install_dir): + zip_archive = ZipFile(cachename, 'r') + try: + zip_archive.extractall(path=install_dir) + return zip_archive.namelist() + except AttributeError: + logger.error("zip extraction not supported by this python version.") + return False + class __SCPOrHTTPHandler(urllib2.BaseHandler): """ Evil hack to allow both the build system and developers consume diff -r ac90b03614ea autobuild/configfile.py --- a/autobuild/configfile.py Mon Aug 22 13:28:27 2011 -0700 +++ b/autobuild/configfile.py Fri Oct 14 11:29:52 2011 -0500 @@ -357,6 +357,7 @@ Describes a dowloadable archive of artifacts for this package. Attributes: + format hash hash_algorithm url @@ -364,6 +365,7 @@ # Implementations for various values of hash_algorithm should be found in # hash_algorithms.py. def __init__(self, dictionary = None): + self.format = None self.hash = None self.hash_algorithm = None self.url = None diff -r ac90b03614ea autobuild/tests/test_common.py --- a/autobuild/tests/test_common.py Mon Aug 22 13:28:27 2011 -0700 +++ b/autobuild/tests/test_common.py Fri Oct 14 11:29:52 2011 -0500 @@ -21,7 +21,12 @@ # $/LicenseInfo$ #!/usr/bin/python +import os +import shutil +import tarfile +import tempfile import unittest +from zipfile import ZipFile from autobuild import common class TestCommon(unittest.TestCase): @@ -35,6 +40,34 @@ exe_path = common.find_executable(shell) assert exe_path != None + + def test_extract_package(self): + try: + tmp_dir = tempfile.mkdtemp() + tar_cachename = common.get_package_in_cache("test.tar.bz2") + zip_cachename = common.get_package_in_cache("test.zip") + test_file_path = os.path.join("data", "package-test", "file2") + + # Test tarball extraction + tar_file = tarfile.open(tar_cachename, 'w:bz2') + tar_file.add(os.path.join(os.path.dirname(__file__), test_file_path), test_file_path) + tar_file.close() + common.extract_package("test.tar.bz2", tmp_dir) + assert os.path.isfile(os.path.join(tmp_dir, "data", "package-test", "file2")) + + # Test zip extraction + zip_archive = ZipFile(zip_cachename, 'w') + zip_archive.write(os.path.join(os.path.dirname(__file__), test_file_path), test_file_path) + zip_archive.close() + common.extract_package("test.zip", tmp_dir) + assert os.path.isfile(os.path.join(tmp_dir, "data", "package-test", "file2")) + finally: + if os.path.isfile(tar_cachename): + os.remove(tar_cachename) + if os.path.isfile(zip_cachename): + os.remove(zip_cachename) + shutil.rmtree(tmp_dir, True) + def tearDown(self): pass diff -r ac90b03614ea autobuild/tests/test_package.py --- a/autobuild/tests/test_package.py Mon Aug 22 13:28:27 2011 -0700 +++ b/autobuild/tests/test_package.py Fri Oct 14 11:29:52 2011 -0500 @@ -30,6 +30,7 @@ import unittest import autobuild.autobuild_tool_package as package from autobuild import configfile +from zipfile import ZipFile class TestPackaging(unittest.TestCase): @@ -38,10 +39,12 @@ data_dir = os.path.join(this_dir, "data") self.config_path = os.path.join(data_dir, "autobuild-package.xml") self.config = configfile.ConfigurationDescription(self.config_path) - self.tar_name = os.path.join(this_dir, "archive-test.tar.bz2") + self.tar_basename = os.path.join(this_dir, "archive-test") + self.tar_name = self.tar_basename + ".tar.bz2" + self.zip_name = self.tar_basename + ".zip" def test_package(self): - package.package(self.config, 'linux', self.tar_name) + package.package(self.config, 'linux', self.tar_basename) assert os.path.exists(self.tar_name) tarball = tarfile.open(self.tar_name) assert [os.path.basename(f) for f in tarball.getnames()].sort() == \ @@ -49,13 +52,20 @@ def test_autobuild_package(self): result = subprocess.call('autobuild package --config-file=%s --archive-name=%s -p linux' % \ - (self.config_path, self.tar_name), shell=True) + (self.config_path, self.tar_basename), shell=True) assert result == 0 assert os.path.exists(self.tar_name) tarball = tarfile.open(self.tar_name) assert [os.path.basename(f) for f in tarball.getnames()].sort() == \ ['file3', 'file1', 'test1.txt'].sort() os.remove(self.tar_name) + result = subprocess.call('autobuild package --config-file=%s --archive-name=%s --archive-format=zip -p linux' % \ + (self.config_path, self.tar_basename), shell=True) + assert result == 0 + assert os.path.exists(self.zip_name) + zip_file = ZipFile(self.zip_name, 'r') + assert [os.path.basename(f) for f in zip_file.namelist()].sort() == \ + ['file3', 'file1', 'test1.txt'].sort() result = subprocess.call('autobuild package --config-file=%s -p linux --dry-run' % \ (self.config_path), shell=True) assert result == 0 @@ -63,6 +73,8 @@ def tearDown(self): if os.path.exists(self.tar_name): os.remove(self.tar_name) + if os.path.exists(self.zip_name): + os.remove(self.zip_name) if __name__ == '__main__': unittest.main()