Use the new commonify script (python instead of ruby). Make sure depends are put on the -[[language]] package if no common package is built. Add -javascript pkg.

3 files changed, 148 insertions, 51 deletions

diff --git a/.SRCINFO b/.SRCINFO
index d309585edfe6..9ffba20a48b1 100644
--- a/.SRCINFO
+++ b/.SRCINFO
@@ -1,21 +1,23 @@
 # Generated by mksrcinfo v8
-# Tue Jun 27 17:28:08 UTC 2017
+# Fri Jun 30 19:41:31 UTC 2017
 pkgbase = eclipse-devel
 	pkgdesc = Highly extensible IDE
 	pkgver = 4.7.0.R
-	pkgrel = 1
+	pkgrel = 2
 	epoch = 1
 	url = https://eclipse.org
 	arch = i686
 	arch = x86_64
 	license = EPL
-	makedepends = ruby
+	makedepends = python
 	noextract = eclipse-cpp-oxygen-R-linux-gtk.tar.gz
 	noextract = eclipse-cpp-oxygen-R-linux-gtk-x86_64.tar.gz
 	noextract = eclipse-jee-oxygen-R-linux-gtk.tar.gz
 	noextract = eclipse-jee-oxygen-R-linux-gtk-x86_64.tar.gz
 	noextract = eclipse-php-oxygen-R-linux-gtk.tar.gz
 	noextract = eclipse-php-oxygen-R-linux-gtk-x86_64.tar.gz
+	noextract = eclipse-javascript-oxygen-R-linux-gtk.tar.gz
+	noextract = eclipse-javascript-oxygen-R-linux-gtk-x86_64.tar.gz
 	noextract = eclipse-java-oxygen-R-linux-gtk.tar.gz
 	noextract = eclipse-java-oxygen-R-linux-gtk-x86_64.tar.gz
 	options = !emptydirs
@@ -28,18 +30,22 @@ pkgbase = eclipse-devel
 	source_i686 = http://ftp-stud.fht-esslingen.de/pub/Mirrors/eclipse/technology/epp/downloads/release/oxygen/R/eclipse-cpp-oxygen-R-linux-gtk.tar.gz
 	source_i686 = http://ftp-stud.fht-esslingen.de/pub/Mirrors/eclipse/technology/epp/downloads/release/oxygen/R/eclipse-jee-oxygen-R-linux-gtk.tar.gz
 	source_i686 = http://ftp-stud.fht-esslingen.de/pub/Mirrors/eclipse/technology/epp/downloads/release/oxygen/R/eclipse-php-oxygen-R-linux-gtk.tar.gz
+	source_i686 = http://ftp-stud.fht-esslingen.de/pub/Mirrors/eclipse/technology/epp/downloads/release/oxygen/R/eclipse-javascript-oxygen-R-linux-gtk.tar.gz
 	source_i686 = http://ftp-stud.fht-esslingen.de/pub/Mirrors/eclipse/technology/epp/downloads/release/oxygen/R/eclipse-java-oxygen-R-linux-gtk.tar.gz
 	sha1sums_i686 = 41fdf3422deba2189dbb4aafc9951c886ec16cb9
 	sha1sums_i686 = c9b8f9f6d2e97d81ed678fbea8047bbfad8287ae
 	sha1sums_i686 = 74602abec77cae9de3ec4456763c35d1add84833
+	sha1sums_i686 = 0d3d890eda8f025efbb6963ad71d00f293f8a352
 	sha1sums_i686 = 23ab70a1c694cb42ed32d4592bbd86a5631571e7
 	source_x86_64 = http://ftp-stud.fht-esslingen.de/pub/Mirrors/eclipse/technology/epp/downloads/release/oxygen/R/eclipse-cpp-oxygen-R-linux-gtk-x86_64.tar.gz
 	source_x86_64 = http://ftp-stud.fht-esslingen.de/pub/Mirrors/eclipse/technology/epp/downloads/release/oxygen/R/eclipse-jee-oxygen-R-linux-gtk-x86_64.tar.gz
 	source_x86_64 = http://ftp-stud.fht-esslingen.de/pub/Mirrors/eclipse/technology/epp/downloads/release/oxygen/R/eclipse-php-oxygen-R-linux-gtk-x86_64.tar.gz
+	source_x86_64 = http://ftp-stud.fht-esslingen.de/pub/Mirrors/eclipse/technology/epp/downloads/release/oxygen/R/eclipse-javascript-oxygen-R-linux-gtk-x86_64.tar.gz
 	source_x86_64 = http://ftp-stud.fht-esslingen.de/pub/Mirrors/eclipse/technology/epp/downloads/release/oxygen/R/eclipse-java-oxygen-R-linux-gtk-x86_64.tar.gz
 	sha1sums_x86_64 = 64b419cea0f1f67cb6cf4a167cbd3a034af6970e
 	sha1sums_x86_64 = c3e8305d5413ef08da1ae89d211d768c914024e0
 	sha1sums_x86_64 = 2bd251a90f803f9a7c2f97ef8aa71932c10c6631
+	sha1sums_x86_64 = f80e0232b7cd090dbd634786042c4be7119b724f
 	sha1sums_x86_64 = 539e2edd7fe72653a7076e696ade52c4a3bafd0d
 
 pkgname = eclipse-common-devel
@@ -48,8 +54,8 @@ pkgname = eclipse-common-devel
 	depends = java-environment>=8
 	depends = webkit2gtk
 	depends = unzip
-	provides = eclipse-common=4.7.0.R-1
-	provides = eclipse-common-devel=4.7.0.R-1
+	provides = eclipse-common=4.7.0.R-2
+	provides = eclipse-common-devel=4.7.0.R-2
 
 pkgname = eclipse-cpp-devel
 
@@ -57,5 +63,7 @@ pkgname = eclipse-jee-devel
 
 pkgname = eclipse-php-devel
 
+pkgname = eclipse-javascript-devel
+
 pkgname = eclipse-java-devel
 
diff --git a/PKGBUILD b/PKGBUILD
index cc0deba80067..a779cc3f8beb 100644
--- a/PKGBUILD
+++ b/PKGBUILD
@@ -13,12 +13,14 @@ _build_java=1
 _build_cpp=1
 _build_jee=1
 _build_php=1
+_build_javascript=1
 
 pkgbase=eclipse-devel
 pkgname=()
-[[ "${_build_cpp}" == '1' ]] && pkgname+=('eclipse-cpp-devel')
-[[ "${_build_jee}" == '1' ]] && pkgname+=('eclipse-jee-devel')
-[[ "${_build_php}" == '1' ]] && pkgname+=('eclipse-php-devel')
+[[ "${_build_cpp}" == '1' ]]        && pkgname+=('eclipse-cpp-devel')
+[[ "${_build_jee}" == '1' ]]        && pkgname+=('eclipse-jee-devel')
+[[ "${_build_php}" == '1' ]]        && pkgname+=('eclipse-php-devel')
+[[ "${_build_javascript}" == '1' ]] && pkgname+=('eclipse-javascript-devel')
 # If no packages are selected, build java by default
 [[ "${_build_java}" == '1' || ${#pkgname[@]} -eq 0 ]] && pkgname+=('eclipse-java-devel')
 # If we're building more than one package, create a common package
@@ -27,14 +29,14 @@ pkgname=()
 epoch=1
 _milestone=R
 pkgver=4.7.0.${_milestone}
-pkgrel=1
+pkgrel=2
 _release=oxygen-${_milestone}
 pkgdesc="Highly extensible IDE"
 license=(EPL)
 arch=(i686 x86_64)
 url="https://eclipse.org"
 makedepends=()
-[[ ${#pkgname[@]} -gt 1 ]] && makedepends+=('ruby')
+[[ ${#pkgname[@]} -gt 1 ]] && makedepends+=('python')
 options=(!emptydirs)
 source=(commonify eclipse.sh eclipse.desktop)
 source_i686=()
@@ -120,6 +122,8 @@ _package() {
   if [[ ${#pkgname[@]} -gt 1 ]] ; then
     depends=(eclipse-common-devel=$pkgver)
   else
+    # If no -common pkg is build, we still need to depend on stuff
+    depends=("java-environment>=8" webkit2gtk unzip)
     conflicts+=('eclipse-common-devel')
   fi
   provides=(eclipse-devel "eclipse-$_variant=$pkgver-$pkgrel")
diff --git a/commonify b/commonify
index 90548697a26d..be2e85399948 100755
--- a/commonify
+++ b/commonify
@@ -1,41 +1,126 @@
-#!/usr/bin/env ruby
-require 'set'
-require 'fileutils'
-
-common = ARGV.shift
-targets = ARGV.dup
-first, *rest = targets
-
-files = targets.map do |dir|
-  Dir.chdir dir
-  g = Dir.glob("**/*").select { |file| test(?f, file) }
-  Dir.chdir ".."
-  puts "#{dir}: Found #{g.size} files"
-  Set.new(g)
-end
-
-commonfiles = files.inject(:&)
-puts "#{commonfiles.size} common files"
-
-samefiles = 0
-commonfiles.each do |file|
-  firstfile = File.join first, file
-  targetfiles = rest.map { |target| File.join target, file }
-
-  is_same = targetfiles.all? do |targetfile|
-    system "diff", "-q", firstfile, targetfile,
-      in: "/dev/null", out: "/dev/null", err: [:child, :out]
-  end
-
-  if is_same
-    commonfile = File.join common, file
-    FileUtils.mkdir_p File.dirname(commonfile)
-    FileUtils.mv firstfile, commonfile
-    FileUtils.rm(targetfiles)
-    samefiles += 1
-  else
-    puts "Divergent common file: #{file}"
-  end
-end
-
-puts "#{common}: #{samefiles} identical common files"
+#!/usr/bin/env python3
+from argparse import ArgumentParser
+from asyncio.subprocess import DEVNULL
+from asyncio import get_event_loop, gather, BoundedSemaphore, create_subprocess_exec
+from enum import Enum, auto
+from logging import getLogger, basicConfig, INFO
+from os import cpu_count
+from pathlib import Path
+from sys import exit
+
+
+logger = getLogger(__name__)
+
+
+class Mode(Enum):
+    identical = auto()
+    nonconflicting = auto()
+
+
+def parse_args(args=None):
+    parser = ArgumentParser(description="Extract common files.")
+
+    modes = parser.add_mutually_exclusive_group(required=True)
+    for m in Mode:
+        modes.add_argument(f"--{m.name}", dest="mode", action="store_const", const=m,
+                           help=f"extract {m.name} common files")
+
+    parser.add_argument("common_dir", metavar="COMMON_DIR",
+                        help="common files directory to move to")
+    parser.add_argument("targets", nargs="+", metavar="INPUT_DIR", help="directory to move from")
+
+    return parser.parse_args(args)
+
+
+diff_semaphore = BoundedSemaphore(value=cpu_count())
+
+
+class DiffError(RuntimeError):
+    def __init__(self, file1, file2):
+        self.file1 = file1
+        self.file2 = file2
+
+
+async def diff(file1, file2):
+    async with diff_semaphore:
+        p = await create_subprocess_exec("diff", "-q", str(file1), str(file2),
+                                         stdin=DEVNULL, stdout=DEVNULL, stderr=DEVNULL)
+        try:
+            ret = await p.wait()
+        except:
+            try:
+                p.kill()
+            except:
+                pass
+
+    if ret != 0:
+        raise DiffError(file1, file2)
+
+
+async def identical(files):
+    if len(files) < 2:
+        return True
+    first, *rest = files
+    fut = gather(*[diff(first, f) for f in rest])
+    try:
+        await fut
+    except DiffError as e:
+        logger.info("Divergent files: %s <> %s", e.file1, e.file2)
+        return False
+    else:
+        return True
+    finally:
+        fut.cancel()
+
+def commonify_file(common_file, files):
+    common_file.parent.mkdir(parents=True, exist_ok=True)
+    first, *rest = files
+    first.rename(common_file)
+    for f in rest:
+        f.unlink()
+
+
+async def commonify_identical(common_file, files):
+    if await identical(files):
+        commonify_file(common_file, files)
+        return 1
+    return 0
+
+
+def arg_dir(s):
+    p = Path(s)
+    if not p.is_dir():
+        raise ValueError(f"{s!r} is not a directory")
+    return p
+
+
+async def main(settings):
+    ignore_len = settings.mode == Mode.nonconflicting
+    common_dir = arg_dir(settings.common_dir)
+    targets = [arg_dir(t) for t in settings.targets]
+
+    d = {}
+    for t in targets:
+        n = 0
+        for f in t.glob("**/*"):
+            if f.is_file():
+                d.setdefault(f.relative_to(t), []).append(f)
+                n += 1
+        logger.info("%s: Found %d files", t, n)
+
+    results = await gather(*[
+        commonify_identical(common_dir / f, tf)
+        for f, tf in d.items()
+        if ignore_len or len(tf) == len(targets)
+    ])
+
+    logger.info("%s: %d %s files",
+                common_dir, sum(results), settings.mode.name)
+    return 0
+
+
+if __name__ == "__main__":
+    basicConfig(level=INFO)
+    settings = parse_args()
+    loop = get_event_loop()
+    exit(loop.run_until_complete(main(settings)))