uhd: Added infrastructure for maintaining images directory

host/CMakeLists.txt

@@ -209,8 +209,12 @@ UHD_INSTALL(FILES
 ########################################################################
 # Images download directory for utils/uhd_images_downloader.py
 ########################################################################
-SET(UHD_IMAGES_MD5SUM "b1e06e7d6fe3eacf49d16e2acca98275")
-SET(UHD_IMAGES_DOWNLOAD_SRC "http://files.ettus.com/binaries/maint_images/archive/uhd-images_003.007.003-release.zip")
+#{{{IMG_SECTION
+# This section is written automatically by /images/create_imgs_package.py
+# Any manual changes in here will be overwritten.
+SET(UHD_IMAGES_MD5SUM "3f4e836c6dfbc714c512662de4855082")
+SET(UHD_IMAGES_DOWNLOAD_SRC "uhd-images_003.007.002-108-gcf5739da.zip")
+#}}}
 
 ########################################################################
 # Register top level components
@@ -329,4 +333,3 @@ ENDIF(LIBUHD_PKG)
 UHD_PRINT_COMPONENT_SUMMARY()
 MESSAGE(STATUS "Building version: ${UHD_VERSION}${PRINT_APPEND}")
 MESSAGE(STATUS "Using install prefix: ${CMAKE_INSTALL_PREFIX}")
-MESSAGE(STATUS "Compatible images can be downloaded from: ${UHD_IMAGES_DOWNLOAD_SRC}")

images/README.md

@@ -3,11 +3,16 @@ UHD Firmware and FPGA Images Builder
 
 The images directory contains the following:
 * A Makefile for building firmware and FPGA images
-* A CMake file for building an images package
+* Scripts to load all the binaries for this current commit, as
+  well as create a new images package from the images in subdir
+  images/
 
 This provides a clean and organized way to build all of the firmware and FPGA
 images, the source code for which is in the `firmware` and `fpga` directories
-one level above this.
+one level above this, and also maintains a linkage between images and git commits.
+
+Building the binaries
+---------------------
 
 The Makefile and build systems for the images are *probably* Unix-specific.
 It's best to build the images on a Unix system with standard build tools.  The
@@ -18,7 +23,55 @@ __To build the images (unix):__
 1. `make clean`
 2. `make images`
 
-__To build the package (unix):__
+__Fedora note:__
+
+The sdcc binaries are prefixed with "sdcc-" which breaks the build.
+However, /usr/libexec/sdcc contains properly named sdcc binaries.
+`export PATH=${PATH}:/usr/libexec/sdcc`
+
+
+Updating binaries
+-----------------
+
+This goes two ways:
+
+1. Loading the correct binaries for this commit
+2. Updating the binary package on this branch
+
+
+### Loading the correct binaries ###
+
+If you check out a branch or commit, you might want to use the exact same
+binaries that were used when this branch or commit was generated.
+To do this, run `populate_images.py`. This will either download the correct
+images package from a web server or from a local directory if
+`UHD_IMAGES_BASE_URL` is set.
+
+### Updating the binaries ###
+
+If you have commited changes to this branch that require new images, you
+should probably update those.
+Simply copy the new image binaries into images/. If necessary, run
+`populate_images.py` before you do any of this, so all the untouched
+images are the correct version. Then, run `create_imgs_package.py --commit "COMMIT MSG"`
+to create a new ZIP file and commit the info.
+If `UHD_IMAGES_BASE_URL` is set and is a local directory, it will move
+the ZIP file to this directory after creating it.
+
+### Updating the binaries ###
+
+Typical workflow:
+
+1. Check out a branch or commit (git checkout FOO)
+2. Update the images/ subdir (`populate_images.py`)
+3. Do some coding, and commit those changes (`git commit`)
+4. Copy new binaries to images/
+5. Commit the new binaries: `create_imgs_package.py --commit "Updated images on branch X"`
+
+### The CPack system ###
+
+Underlying `create_imgs_package.py` is a CPack system, which can be manually
+invoked by:
 
 1. `mkdir build`
 2. `cd build`
@@ -28,8 +81,3 @@ __To build the package (unix):__
 The package generator types are described here:
 http://www.cmake.org/Wiki/CMake:CPackPackageGenerators
 
-__Fedora note:__
-
-The sdcc binaries are prefixed with "sdcc-" which breaks the build.
-However, /usr/libexec/sdcc contains properly named sdcc binaries.
-`export PATH=${PATH}:/usr/libexec/sdcc`

images/create_imgs_package.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+#
+# Copyright 2014 Ettus Research LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+"""
+Command-line utility to create a .zip-file with the current image set.
+"""
+
+import re
+import os
+import uhdimgs
+import glob
+import subprocess
+import argparse
+import shutil
+
+def clear_img_dir(img_root_dir):
+    """ Removes non-image files from the images dir """
+    globs = ["*.tag", "LICENSE"]
+    for the_glob in globs:
+        for filename in glob.iglob(os.path.join(img_root_dir, the_glob)):
+            print 'Removing file from images directory: ', filename
+            os.unlink(filename)
+
+def get_zipfilename_from_cpack_output(cpoutput):
+    """ Parses the output of the ZIP-file creating script
+    and scrapes the actual file name. """
+    regex = re.compile("\/build\/(?P<filename>.*\.zip)")
+    results = regex.search(cpoutput)
+    return results.group('filename')
+
+def parse_args():
+    """ Parse args, duh """
+    parser = argparse.ArgumentParser(description='Link the current set of images to this commit.')
+    parser.add_argument('--commit', default=None,
+                       help='Supply a commit message to the changes to host/CMakeLists.txt.')
+    parser.add_argument('-r', '--release-mode', default=None,
+                       help='Specify UHD_RELEASE_MODE. Typically "release" or "rc1" or similar.')
+    return parser.parse_args()
+
+def move_zip_to_repo(base_url, zipfilename):
+    final_destination = os.path.join(base_url, zipfilename)
+    if os.path.exists(final_destination):
+        print "WARNING: A file with name {} is already in the images repository.".format(zipfilename)
+        print "Overwrite? [y/N]",
+        ans = raw_input()
+        if ans.strip().upper() != 'Y':
+            return
+        os.unlink(final_destination)
+    shutil.move(zipfilename, base_url)
+
+def main():
+    " Go, go, go! "
+    args = parse_args()
+    img_root_dir = os.path.join(uhdimgs.get_images_dir(), 'images')
+    os.chdir(uhdimgs.get_images_dir())
+    print "== Clearing out the images directory..."
+    clear_img_dir(img_root_dir)
+    print "== Creating ZIP file..."
+    cpack_cmd = ["./make_zip.sh",]
+    if args.release_mode is not None:
+        cpack_cmd.append(args.release_mode)
+    try:
+        cpack_output = subprocess.check_output(cpack_cmd)
+    except subprocess.CalledProcessError as e:
+        print e.output
+        raise SystemExit, 1
+    zipfilename = get_zipfilename_from_cpack_output(cpack_output)
+    print "Filename: ", zipfilename
+    print "== Calculating MD5 sum of ZIP archive..."
+    md5 = uhdimgs.md5_checksum(zipfilename)
+    print 'MD5: ', md5
+    base_url = uhdimgs.get_base_url()
+    if uhdimgs.base_url_is_local(base_url) and os.access(base_url, os.W_OK):
+        print "== Moving ZIP file to {}...".format(base_url)
+        move_zip_to_repo(base_url, zipfilename)
+    print "== Updating CMakeLists.txt..."
+    uhdimgs.update_main_cmake_file(md5, zipfilename)
+    if args.commit is not None:
+        print "== Committing changes..."
+        subprocess.check_call(['git', 'commit', '-m', args.commit, uhdimgs.get_cmake_main_file()])
+    print "== Done!"
+
+if __name__ == "__main__":
+    main()

images/make_zip.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+# Automatically run the make-zip-file process
+if [ ! -e 'make_zip.sh' ]; then
+    echo 'Are you running this from the images/ directory?'
+    exit 1
+fi
+if [ -e 'build' ]; then
+    echo 'Please remove build subdirectory before proceeding.'
+    exit 1
+fi
+mkdir build
+cd build
+cmake .. -DCPACK_GENERATOR=ZIP -DUHD_RELEASE_MODE="$1" ..
+make package
+mv uhd-images*.zip ..
+cd ..
+rm -r build

images/populate_images.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+#
+# Copyright 2014 Ettus Research LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+"""
+Populates the current directory with a valid set of binaries for the
+current commit.
+"""
+
+import os
+import re
+import subprocess
+import uhdimgs
+
+def get_md5_and_zipfilename():
+    """ Return MD5 hash and ZIP filename from the host/CMakeLists.txt file. """
+    cmakef = open(uhdimgs.get_cmake_main_file(), 'r').read()
+    md5_regex = re.compile(r'UHD_IMAGES_MD5SUM\s*"(?P<md5>[0-9a-f]{32})', flags=re.MULTILINE)
+    md5 = md5_regex.search(cmakef).groups('md5')[0]
+    filename_regex = re.compile(r'UHD_IMAGES_DOWNLOAD_SRC\s*"(?P<filename>[^"]*\.zip)', flags=re.MULTILINE)
+    filename = filename_regex.search(cmakef).groups('filename')[0]
+    return (md5, filename)
+
+def main():
+    " Go, go, go! "
+    # Switch to correct dir
+    img_root_dir = os.path.join(uhdimgs.get_images_dir(), 'images')
+    os.chdir(uhdimgs.get_images_dir())
+    # Read out the CMakeLists.txt file, get filename
+    print "== Reading MD5 and ZIP filename for current commit from {}...".format(uhdimgs.get_cmake_main_file())
+    (md5, filename) = get_md5_and_zipfilename()
+    print "== Starting download..."
+    try:
+        downloader_cmd = [
+                'python',
+                '../host/utils/uhd_images_downloader.py.in',
+                '-i', img_root_dir,
+                '-f', filename,
+                '-c', md5
+        ]
+        subprocess.check_call(downloader_cmd)
+    except (subprocess.CalledProcessError, OSError):
+        print "[ERROR] Failed to run downloader script."
+        exit(1)
+    print "== Done!"
+
+if __name__ == "__main__":
+    main()

images/uhdimgs.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+#
+# Copyright 2014 Ettus Research LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+"""
+Utility module for packaging and handling UHD binary images.
+"""
+
+import re
+import os
+import sys
+import hashlib
+
+_DEFAULT_BASE_URL = "http://files.ettus.com/binaries/images"
+_DEFAULT_BUFFER_SIZE      = 8192
+_CMAKE_MAIN_FILE = "../host/CMakeLists.txt"
+_CMAKE_IMG_TPL = """
+# This section is written automatically by /images/create_imgs_package.py
+# Any manual changes in here will be overwritten.
+SET(UHD_IMAGES_MD5SUM "{}")
+SET(UHD_IMAGES_DOWNLOAD_SRC "{}")
+"""
+
+def get_cmake_main_file():
+    """ Returns a path to the CMakeLists.txt file that contains the image info. """
+    return _CMAKE_MAIN_FILE
+
+def get_base_url():
+    """ Returns the base URL """
+    if os.environ.get("UHD_IMAGES_BASE_URL") is not None and os.environ.get("UHD_IMAGES_BASE_URL") != "":
+        return os.environ.get("UHD_IMAGES_BASE_URL")
+    else:
+        return _DEFAULT_BASE_URL
+
+def base_url_is_local(base_url):
+    """ Returns true if the base URL is actually a http URL
+    instead of a local path. """
+    return not (base_url.find('http') == 0)
+
+def get_images_dir():
+    """
+    Returns the absolute position of the images/ subdir
+    in the UHD source tree.
+    """
+    return os.path.dirname(__file__)
+
+def update_main_cmake_file(md5, zipfilename):
+    """
+    Update the section in host/CMakeLists.txt that contains
+    the ZIP filename and the MD5 hash.
+    """
+    cmakef = open(_CMAKE_MAIN_FILE, 'r').read()
+    new_section = _CMAKE_IMG_TPL.format(md5, zipfilename)
+    regex = re.compile("(?<={{{IMG_SECTION)(.*)(?=#}}})", flags=re.MULTILINE|re.DOTALL)
+    cmakef = regex.sub(new_section, cmakef, count=1)
+    open(_CMAKE_MAIN_FILE, 'w').write(cmakef)
+
+def get_total_md5(img_dir):
+    """ Creates an md5sum of everything in the images directory """
+    def _update_md5_for_dir_recursive(dir_root, md5_obj):
+        for (root, dirnames, filenames) in os.walk(dir_root):
+            for filename in filenames:
+                md5_obj.update(open(os.path.join(root, filename), 'rb').read())
+                sys.stdout.write('.')
+                sys.stdout.flus()
+            for dirname in dirnames:
+                _update_md5_for_dir_recursive(os.path.join(root, dirname), md5_obj)
+    md5 = hashlib.md5()
+    _update_md5_for_dir_recursive(img_dir, md5)
+    print ""
+    return md5.hexdigest()
+
+def md5_checksum(filePath):
+    """ Return MD5 checksum of a single file. """
+    try:
+        with open(filePath, 'rb') as fh:
+            m = hashlib.md5()
+            while True:
+                data = fh.read(_DEFAULT_BUFFER_SIZE)
+                if not data:
+                    break
+                m.update(data)
+            return m.hexdigest()
+    except Exception, e:
+        print "Failed to calculated MD5 sum of: %s (%s)" % (filePath, e)
+        raise e