ros-build-tools: start adding Python 3 support.

4 files changed, 100 insertions, 27 deletions

diff --git a/.SRCINFO b/.SRCINFO
index 8997aa8eca0c..5655c2b06349 100644
--- a/.SRCINFO
+++ b/.SRCINFO
@@ -1,7 +1,7 @@
 pkgbase = ros-build-tools
 	pkgdesc = Utilities for building arch packages for ROS stacks.
-	pkgver = 0.0.6
-	pkgrel = 2
+	pkgver = 0.0.7
+	pkgrel = 1
 	url = http://www.ros.org
 	arch = i686
 	arch = x86_64
@@ -19,14 +19,14 @@ pkgbase = ros-build-tools
 	source = generate-python-patch.sh
 	source = import_catkin_packages.py
 	source = clear-ros-env.sh
-	md5sums = 8b2e864de97e97298ebfb8da471f2df1
+	md5sums = b30a6d9a49bf50fad3db45b8753fb17a
 	md5sums = 79ae7fb600e116623a42631d15d66a87
 	md5sums = ac82eca7efc9f0ff7e8b976a83692868
 	md5sums = f3378832c3ba121f7c9e17dc43c8b1d4
 	md5sums = d257f7f20384e894b0431ee61068aa96
 	md5sums = 563c9d1320a3a997db25d3087303dcfb
 	md5sums = 8d6d7eb89a12c449497b209f1a06655b
-	md5sums = 77844d066c1aedf9b3373ad9ab81fcaa
+	md5sums = 5a0176cd0a668e728a9aefd80442d8b7
 	md5sums = 07f5253eb3f8cb5295c32026a20ab6c0
 
 pkgname = ros-build-tools
diff --git a/PKGBUILD b/PKGBUILD
index 65845178f328..c5dde4f6baa3 100644
--- a/PKGBUILD
+++ b/PKGBUILD
@@ -4,9 +4,9 @@ pkgdesc='Utilities for building arch packages for ROS stacks.'
 url=http://www.ros.org
 
 pkgname='ros-build-tools'
-pkgver='0.0.6'
+pkgver='0.0.7'
 arch=('i686' 'x86_64')
-pkgrel=2
+pkgrel=1
 license=('BSD')
 makedepends=()
 depends=()
@@ -34,12 +34,12 @@ package() {
     done
 }
 
-md5sums=('8b2e864de97e97298ebfb8da471f2df1'
+md5sums=('b30a6d9a49bf50fad3db45b8753fb17a'
          '79ae7fb600e116623a42631d15d66a87'
          'ac82eca7efc9f0ff7e8b976a83692868'
          'f3378832c3ba121f7c9e17dc43c8b1d4'
          'd257f7f20384e894b0431ee61068aa96'
          '563c9d1320a3a997db25d3087303dcfb'
          '8d6d7eb89a12c449497b209f1a06655b'
-         '77844d066c1aedf9b3373ad9ab81fcaa'
+         '5a0176cd0a668e728a9aefd80442d8b7'
          '07f5253eb3f8cb5295c32026a20ab6c0')
diff --git a/fix-python-scripts.sh b/fix-python-scripts.sh
index f0fb79f8309d..de6265285674 100755
--- a/fix-python-scripts.sh
+++ b/fix-python-scripts.sh
@@ -1,18 +1,49 @@
 #!/bin/bash
 
 if [ -z "$1" ]; then
-    echo "Usage: $0 <directory>"
+    echo "Usage: $0 [-v/--version PYTHON_VERSION] <directory>"
     echo ""
-    echo "Makes sure that all python scripts use the python2 command"
+    echo "Makes sure that all Python scripts use the right python command."
+    echo "PYTHON_VERSION: either 2 or 3 (default = 2)."
+    echo
+    echo "Note that according to PEP 394, developers should use \"python\" in
+          the shebang line for code compatible with both Python 2 and 3, but
+          since this may not be the case, we always overwrite the shebang line.
+          For more information: http://legacy.python.org/dev/peps/pep-0394/"
     exit 1
 fi
 
+# Default Python version: 2
+PYTHON_VERSION=2
+
+while [[ $# > 1 ]]
+do
+  key="$1"
+  shift
+
+  case $key in
+    -v|--version)
+      PYTHON_VERSION="$1"
+      shift
+      ;;
+    *)
+      # unknown option
+      ;;
+  esac
+done
+
+# Check user input
+if [[ "$PYTHON_VERSION" != "2" && "$PYTHON_VERSION" != "3" ]]; then
+  echo "Error: invalid Python version given: $PYTHON_VERSION"
+  exit 2
+fi
+
 for file in $(grep -rl -e 'env python *$' -e 'bin/python *$' $1); do
     if [ -z "$file" ]; then
         echo "Error finding files."
         exit 1
     fi
-    sed -i 's,env python *$,env python2,g' $file
-    sed -i 's,/usr/bin/python *$,/usr/bin/env python2,g' $file
+    sed -i "s,env python *$,env python${PYTHON_VERSION},g" $file
+    sed -i "s,/usr/bin/python *$,/usr/bin/env python${PYTHON_VERSION},g" $file
 done
 
diff --git a/import_catkin_packages.py b/import_catkin_packages.py
index 1fbe6fb2b452..c2f04832933d 100755
--- a/import_catkin_packages.py
+++ b/import_catkin_packages.py
@@ -18,7 +18,7 @@ import pickle
 
 updates_packages_dir = "/tmp/import_catkin_packages"
 updated_packages_file = os.path.join(updates_packages_dir,
-                                     "updated_packages.dump")
+                                     "updated_packages_%(distro)s.dump")
 
 class PackageBase(object):
 
@@ -35,6 +35,7 @@ class PackageBase(object):
     self.licenses = package.licenses
     self.run_dependencies = list(OrderedDict.fromkeys([dependency.name for dependency in package.run_depends]))
     self.build_dependencies = list(OrderedDict.fromkeys([dependency.name for dependency in package.build_depends + package.buildtool_depends]))
+
     # This may be the case for some metapackages
     self.is_virtual = False
 
@@ -217,16 +218,16 @@ build() {
   cd ${srcdir}/build
 
   # Fix Python2/Python3 conflicts
-  /usr/share/ros-build-tools/fix-python-scripts.sh ${srcdir}/${_dir}
+  /usr/share/ros-build-tools/fix-python-scripts.sh -v %(python_version_major)s ${srcdir}/${_dir}
 
   # Build project
   cmake ${srcdir}/${_dir} \\
         -DCMAKE_BUILD_TYPE=Release \\
         -DCATKIN_BUILD_BINARY_PACKAGE=ON \\
         -DCMAKE_INSTALL_PREFIX=/opt/ros/%(distro)s \\
-        -DPYTHON_EXECUTABLE=/usr/bin/python2 \\
-        -DPYTHON_INCLUDE_DIR=/usr/include/python2.7 \\
-        -DPYTHON_LIBRARY=/usr/lib/libpython2.7.so \\
+        -DPYTHON_EXECUTABLE=%(python_executable)s \\
+        -DPYTHON_INCLUDE_DIR=%(python_include_dir)s \\
+        -DPYTHON_LIBRARY=%(python_library)s \\
         -DSETUPTOOLS_DEB_LAYOUT=OFF
   make
 }
@@ -237,7 +238,7 @@ package() {
 }
 """
 
-  def generate(self, exclude_dependencies=[], rosdep_urls=[]):
+  def generate(self, python_version, exclude_dependencies=[], rosdep_urls=[]):
     raw_build_dep, raw_run_dep = self._get_ros_dependencies()
     ros_build_dep = [dependency for dependency in raw_build_dep
                      if dependency not in exclude_dependencies]
@@ -250,6 +251,12 @@ package() {
     other_run_dep = [dependency for dependency in other_raw_run_dep
                    if dependency not in exclude_dependencies]
 
+    python_version_major = python_version.split('.')[0]
+    python_version_include = python_version
+    # Python 3 include directory is /usr/include/python3.4m... Because why not?
+    if python_version_major == "3":
+      python_version_include = "%s%s" % (python_version_include, "m")
+
     pkgbuild = self.BUILD_TEMPLATE % {
       'distro': self.distro.name,
       'arch_package_name': self._rosify_package_name(self.name),
@@ -264,7 +271,11 @@ package() {
       'ros_build_dependencies': '\n  '.join(ros_build_dep),
       'ros_run_dependencies': '\n  '.join(ros_run_dep),
       'other_build_dependencies': '\n  '.join(other_build_dep),
-      'other_run_dependencies': '\n  '.join(other_run_dep)
+      'other_run_dependencies': '\n  '.join(other_run_dep),
+      'python_version_major': python_version_major,
+      'python_executable': '/usr/bin/python%s' % python_version_major,
+      'python_include_dir': '/usr/include/python%s' % python_version_include,
+      'python_library': '/usr/lib/libpython%s.so' % python_version
       }
 
     # Post-processing:
@@ -300,7 +311,8 @@ md5sums=()
 
   def __init__(self, distro, repository_url, name, version, version_patch):
     try:
-      super(MetaPackage, self).__init__(distro, repository_url, name, version, version_patch)
+      super(MetaPackage, self).__init__(distro, repository_url, name, version,
+                                        version_patch)
     except urllib2.HTTPError:
       # Virtual metapackage
       # TODO: there should be a cleaner way to deal with this...
@@ -346,11 +358,12 @@ md5sums=()
 
 class DistroDescription(object):
 
-  def __init__(self, name, url):
+  def __init__(self, name, url, python_version):
     stream = urllib2.urlopen(url)
     self.name = name
     self._distro = yaml.load(stream)
     self._package_cache = {}
+    self.python_version = python_version
     if self.name == "fuerte":
       if self.name != self._distro['release-name']:
         raise Exception('ROS distro names do not match (%s != %s)' % (self.name, self._distro['release-name']))
@@ -534,12 +547,13 @@ def generate_pkgbuild(distro, package, directory, force=False,
            colored(package.version + '-' + package.version_patch, 'white',
                    attrs=['bold'])))
   with open(pkgbuild_file, 'w') as pkgbuild:
-    pkgbuild.write(package.generate(exclude_dependencies, rosdep_urls))
+    pkgbuild.write(package.generate(distro.python_version, exclude_dependencies,
+                                    rosdep_urls))
 
 
 def main():
   parser = OptionParser(usage='usage: %prog [options] PACKAGE...')
-  parser.add_option('--distro', default='groovy', metavar='distro',
+  parser.add_option('--distro', default='hydro', metavar='distro',
                     help='Select the ROS distro to use.')
   parser.add_option('--list-packages', dest='list_packages', action='store_true',
                     default=False, help='Lists all available packages.')
@@ -562,6 +576,9 @@ def main():
                     default='',
                     help='Comma-separated list of (source) package dependencies'
                     ' to exclude from the generated PKGBUILD file.')
+  parser.add_option('--python-version', metavar='python_version', default='',
+                    help='Python version that will be used. Accepted values are'
+                    ' 2.7 or 3. Note that Python 3 is only supported in Indigo.')
   parser.add_option('-f', '--force', dest='force', action='store_true',
                     default=False,
                     help='Always overwrite exiting PKGBUILD files.')
@@ -580,10 +597,34 @@ def main():
     # Use legagy fuerte URL
     options.distro_url = 'https://raw.github.com/ros/rosdistro/master/releases/%s.yaml'
 
+  # Dictionary containing valid Python versions
+  valid_python_versions = {"fuerte": ["2.7"],
+                           "groovy": ["2.7"],
+                           "hydro":  ["2.7"],
+                           "indigo": ["2.7", "3.4"]}
+
+  # Default Python version that will be used
+  default_python_version = {"fuerte": "2.7",
+                            "groovy": "2.7",
+                            "hydro":  "2.7",
+                            "indigo": "3.4"}
+
+  python_version = default_python_version[options.distro]
+  if options.python_version != "":
+    if options.python_version in valid_python_versions[options.distro]:
+      python_version = options.python_version
+    else:
+      print("Invalid Python version (%s) for %s, using version %s instead."
+            % options.python_version % options.distro % python_version)
+
   distro = DistroDescription(options.distro,
+                             python_version=python_version,
                              url=options.distro_url % options.distro)
 
   if options.output_directory:
+    if not os.path.exists(options.output_directory):
+      os.makedirs(options.output_directory)
+
     if os.path.isdir(options.output_directory):
       distro_dir = os.path.abspath(options.output_directory)
     else:
@@ -600,12 +641,13 @@ def main():
       print("Missing mandatory --output-directory. Exiting.")
       sys.exit()
     generated = set()
-    if os.path.isfile(updated_packages_file):
+    distro_dump_file = updated_packages_file % {'distro': options.distro}
+    if os.path.isfile(distro_dump_file):
       # Load dump of already updated packages to speedup updates
       print('Loading set of previously updated packages: %s'
-            % (colored(updated_packages_file, 'white',
+            % (colored(distro_dump_file, 'white',
                        attrs=['bold'])))
-      updated_packages = open(updated_packages_file, "r")
+      updated_packages = open(distro_dump_file, "r")
       generated = pickle.load(updated_packages)
       updated_packages.close()
       for package in sorted(generated):
@@ -620,7 +662,7 @@ def main():
                         rosdep_urls=options.rosdep_urls, generated=generated)
     if not os.path.exists(updates_packages_dir):
       os.makedirs(updates_packages_dir)
-    updated_packages = open(updated_packages_file, "w+")
+    updated_packages = open(distro_dump_file, "w+")
     pickle.dump(generated, updated_packages)
     updated_packages.close()
   else: