massive rewrite to implant @utuhiro78's scripts

- passing shellcheck
- update mozc revision

2 files changed, 99 insertions, 47 deletions

diff --git a/.SRCINFO b/.SRCINFO
index 762248d56a92..d6aaa31f7a92 100644
--- a/.SRCINFO
+++ b/.SRCINFO
@@ -1,24 +1,26 @@
 pkgbase = mozc-ut-full
-	pkgver = 2.28.4960.102.20221022
+	pkgver = 2.28.4990.102.20230113
 	pkgrel = 1
 	url = https://github.com/fcitx/mozc
 	arch = x86_64
 	license = custom
-	makedepends = pkg-config
+	makedepends = bazel
+	makedepends = git
 	makedepends = python
+	makedepends = python-six
+	makedepends = pkg-config
 	makedepends = curl
 	makedepends = gtk2
 	makedepends = mesa
-	makedepends = ninja
-	makedepends = git
+	makedepends = subversion
+	makedepends = qt5-base
 	makedepends = clang
-	makedepends = python-six
 	makedepends = fcitx5
 	makedepends = emacs
-	makedepends = qt5-base
 	makedepends = ibus
-	makedepends = gyp
-	source = git+https://github.com/fcitx/mozc.git#commit=d4a26a6
+	makedepends = ruby
+	noextract = jawiki-latest-all-titles-in-ns0.gz
+	source = git+https://github.com/fcitx/mozc.git#commit=d42831f
 	source = https://osdn.net/projects/ponsfoot-aur/storage/mozc/x-ken-all-202110.zip
 	source = https://osdn.net/projects/ponsfoot-aur/storage/mozc/jigyosyo-202110.zip
 	source = git+https://github.com/abseil/abseil-cpp.git
@@ -28,8 +30,16 @@ pkgbase = mozc-ut-full
 	source = git+https://github.com/hiroyuki-komatsu/japanese-usage-dictionary.git
 	source = git+https://github.com/open-source-parsers/jsoncpp.git
 	source = git+https://github.com/google/protobuf.git
-	source = https://osdn.net/users/utuhiro/pf/utuhiro/dl/mozcdic-ut-20221022.tar.bz2
-	validpgpkeys = 2CC8A0609AD2A479C65B6D5C8E8B898CBF2412F9
+	source = git+https://github.com/utuhiro78/merge-ut-dictionaries.git
+	source = https://dumps.wikimedia.org/jawiki/latest/jawiki-latest-all-titles-in-ns0.gz
+	source = https://github.com/utuhiro78/mozcdic-ut-alt-cannadic/raw/main/mozcdic-ut-alt-cannadic.txt.tar.bz2
+	source = https://github.com/utuhiro78/mozcdic-ut-edict2/raw/main/mozcdic-ut-edict2.txt.tar.bz2
+	source = https://github.com/utuhiro78/mozcdic-ut-jawiki/raw/main/mozcdic-ut-jawiki.txt.tar.bz2
+	source = https://github.com/utuhiro78/mozcdic-ut-neologd/raw/main/mozcdic-ut-neologd.txt.tar.bz2
+	source = https://github.com/utuhiro78/mozcdic-ut-personal-names/raw/main/mozcdic-ut-personal-names.txt.tar.bz2
+	source = https://github.com/utuhiro78/mozcdic-ut-place-names/raw/main/mozcdic-ut-place-names.txt.tar.bz2
+	source = https://github.com/utuhiro78/mozcdic-ut-skk-jisyo/raw/main/mozcdic-ut-skk-jisyo.txt.tar.bz2
+	source = https://github.com/utuhiro78/mozcdic-ut-sudachidict/raw/main/mozcdic-ut-sudachidict.txt.tar.bz2
 	sha512sums = SKIP
 	sha512sums = dec6479b42ddc1355cd882d17824cd874d8f103ad7767bac3f490f04551059d65b2806fa9e3f39a50ced2ecfdd37b75c9ed4536d9ad3bcef9e8c5ae1ec10e302
 	sha512sums = 606f45d48a9dad0e80a566cab0001910de3c6b2f634ec52c6ef6f44745b55ae8e181b3e3cdf90525a08be1f180eb35900672c90c6ab4f43679a178e863378bbc
@@ -40,7 +50,16 @@ pkgbase = mozc-ut-full
 	sha512sums = SKIP
 	sha512sums = SKIP
 	sha512sums = SKIP
-	sha512sums = 08f8fe1757fd69c94d6d46a7f6cdd2398a6d946cb69531131697f5a35fdb970742fb26aa4285a2025ab14a6fe2992f4f1ec299d20b899db98cad1c53be9a0ca3
+	sha512sums = SKIP
+	sha512sums = 53e45c34f76b9202fb4afef985b442b9c25c8ffcaf004e0fa848a9bd85f9b413afc1feee3789f9b9802f0288137e54caa9929ace94561799e3d5c1f283d4ec72
+	sha512sums = 30019a9ce73456046f67edd6fe8f4661bd9a8e9ca201f3bdf22d2fa70dad9544bd595a8820fbed402a0709809d02cabbdea9dc79ee1f5bf30f8ef722ba4a2c17
+	sha512sums = 8ce42d2eb8bda0e79dbe97467621d48615d6cf0296c99d4d99a4dfc755753d6c99392606a0dc94c9472a23d72feb1c8179d0baf601f01a0e85938125887935b2
+	sha512sums = 33beeb84c5fe7ff8717e2e7e8650f2b33a1920564e363776ff357b94c6eb4f9b8d7c34decb8a3549075a1cc41a2a8d406cd2b07f34cdf5cfe3491cfedf2312d5
+	sha512sums = 990525bad91b47c39775a407f07c2ab00610b9380e0dae200e27665ae7c558885ffe4d7a8666813faf59fc9280e57b666c1edf328e03fbd6492357e25899ce4e
+	sha512sums = af436f5795abab05392456cf489ced282c268d1345095a6e414f47f440b899638aec482a862c1b6c7d331ab724a0e9c02165dfac4bc61d11f11858a0326cf0da
+	sha512sums = 6c19befc6ad34d4924d1fb9f00704337274b19a8c34dc1124167f5c57d3c50d89d2f7382aeadb19a076c13d5e4ba5079f57b488854a6ad80df08e429c93ddb2c
+	sha512sums = 5fc21d5d60fb504eb8474d1b72082c9d14c8864ffc4fa77e157219d2504ebd18778c4f268f12798b2cf03980b9c3bf1c39dfc5a3e63abf929bf586de1375ebb7
+	sha512sums = dd2ff7e4ee3e6ef4ffc415a506ceb4a2f47976be8afda6bdb60a2830890e8b09c6c456b8ac67aa6aa8327bf0a11fb22039a80621e4cbbb145099b86ef87d7359
 
 pkgname = mozc-ut-full-common
 	pkgdesc = A Japanese Input Method for Chromium OS, Windows, Mac and Linux (the Open Source Edition of Google Japanese Input)
diff --git a/PKGBUILD b/PKGBUILD
index 7fe519b061a2..3ae4723ff372 100644
--- a/PKGBUILD
+++ b/PKGBUILD
@@ -1,29 +1,38 @@
 # $Id$
 # Maintainer: BrLi <brli at chakralinux dot org>
+# Contributor: Jiachen Yang <farseerfc@archlinux.org>
 # Contributor: Felix Yan <felixonmars@archlinux.org>
 # Contributor: ponsfoot <cabezon dot hashimoto at gmail dot com>
 # Contributor: UTUMI Hirosi <utuhiro78 at yahoo dot co dot jp>
 
 ## Mozc compile option
 _bldtype='Release'
-_mozc_commit=d4a26a6
+_mozc_commit=d42831f
 _zipcode_rel=202110
 
 # Ut Dictionary
-_utdicdate=20221022
+_utdicdate=20230113
+_dict=(alt-cannadic
+       edict2
+       jawiki
+       neologd
+       personal-names
+       place-names
+       skk-jisyo
+       sudachidict)
 
 _pkgbase=mozc-ut
 pkgbase=$_pkgbase-full
-pkgname=(${pkgbase}-common ibus-${pkgbase} fcitx5-${pkgbase} emacs-${pkgbase})
-pkgver=2.28.4960.102.20221022
+pkgname=("$pkgbase-common" "ibus-$pkgbase" "fcitx5-$pkgbase" "emacs-$pkgbase")
+pkgver=2.28.4990.102.20230113
 pkgrel=1
 arch=('x86_64')
 url="https://github.com/fcitx/mozc"
 license=('custom')
-makedepends=('pkg-config' 'python' 'curl' 'gtk2' 'mesa' 'ninja' 'git' 'clang' 'python-six' 'fcitx5' 'emacs' 'qt5-base' 'ibus' 'gyp')
-source=(git+https://github.com/fcitx/mozc.git#commit=${_mozc_commit}
-        https://osdn.net/projects/ponsfoot-aur/storage/mozc/x-ken-all-${_zipcode_rel}.zip
-        https://osdn.net/projects/ponsfoot-aur/storage/mozc/jigyosyo-${_zipcode_rel}.zip
+makedepends=('bazel' 'git' 'python' 'python-six' 'pkg-config' 'curl' 'gtk2' 'mesa' 'subversion' 'qt5-base' 'clang' 'fcitx5' 'emacs' 'ibus' 'ruby')
+source=(git+https://github.com/fcitx/mozc.git#commit="${_mozc_commit}"
+        "https://osdn.net/projects/ponsfoot-aur/storage/mozc/x-ken-all-${_zipcode_rel}.zip"
+        "https://osdn.net/projects/ponsfoot-aur/storage/mozc/jigyosyo-${_zipcode_rel}.zip"
         git+https://github.com/abseil/abseil-cpp.git
         git+https://chromium.googlesource.com/breakpad/breakpad
         git+https://github.com/google/googletest.git
@@ -31,7 +40,13 @@ source=(git+https://github.com/fcitx/mozc.git#commit=${_mozc_commit}
         git+https://github.com/hiroyuki-komatsu/japanese-usage-dictionary.git
         git+https://github.com/open-source-parsers/jsoncpp.git
         git+https://github.com/google/protobuf.git
-        https://osdn.net/users/utuhiro/pf/utuhiro/dl/mozcdic-ut-${_utdicdate}.tar.bz2)
+        git+https://github.com/utuhiro78/merge-ut-dictionaries.git
+        https://dumps.wikimedia.org/jawiki/latest/jawiki-latest-all-titles-in-ns0.gz)
+noextract=(jawiki-latest-all-titles-in-ns0.gz)
+
+for dict in "${_dict[@]}"; do
+  source+=("https://github.com/utuhiro78/mozcdic-ut-${dict}/raw/main/mozcdic-ut-${dict}.txt.tar.bz2")
+done
 sha512sums=('SKIP'
             'dec6479b42ddc1355cd882d17824cd874d8f103ad7767bac3f490f04551059d65b2806fa9e3f39a50ced2ecfdd37b75c9ed4536d9ad3bcef9e8c5ae1ec10e302'
             '606f45d48a9dad0e80a566cab0001910de3c6b2f634ec52c6ef6f44745b55ae8e181b3e3cdf90525a08be1f180eb35900672c90c6ab4f43679a178e863378bbc'
@@ -42,20 +57,25 @@ sha512sums=('SKIP'
             'SKIP'
             'SKIP'
             'SKIP'
-            '08f8fe1757fd69c94d6d46a7f6cdd2398a6d946cb69531131697f5a35fdb970742fb26aa4285a2025ab14a6fe2992f4f1ec299d20b899db98cad1c53be9a0ca3')
-validpgpkeys=('2CC8A0609AD2A479C65B6D5C8E8B898CBF2412F9')  # Weng Xuetian
+            'SKIP'
+            '53e45c34f76b9202fb4afef985b442b9c25c8ffcaf004e0fa848a9bd85f9b413afc1feee3789f9b9802f0288137e54caa9929ace94561799e3d5c1f283d4ec72'
+            '30019a9ce73456046f67edd6fe8f4661bd9a8e9ca201f3bdf22d2fa70dad9544bd595a8820fbed402a0709809d02cabbdea9dc79ee1f5bf30f8ef722ba4a2c17'
+            '8ce42d2eb8bda0e79dbe97467621d48615d6cf0296c99d4d99a4dfc755753d6c99392606a0dc94c9472a23d72feb1c8179d0baf601f01a0e85938125887935b2'
+            '33beeb84c5fe7ff8717e2e7e8650f2b33a1920564e363776ff357b94c6eb4f9b8d7c34decb8a3549075a1cc41a2a8d406cd2b07f34cdf5cfe3491cfedf2312d5'
+            '990525bad91b47c39775a407f07c2ab00610b9380e0dae200e27665ae7c558885ffe4d7a8666813faf59fc9280e57b666c1edf328e03fbd6492357e25899ce4e'
+            'af436f5795abab05392456cf489ced282c268d1345095a6e414f47f440b899638aec482a862c1b6c7d331ab724a0e9c02165dfac4bc61d11f11858a0326cf0da'
+            '6c19befc6ad34d4924d1fb9f00704337274b19a8c34dc1124167f5c57d3c50d89d2f7382aeadb19a076c13d5e4ba5079f57b488854a6ad80df08e429c93ddb2c'
+            '5fc21d5d60fb504eb8474d1b72082c9d14c8864ffc4fa77e157219d2504ebd18778c4f268f12798b2cf03980b9c3bf1c39dfc5a3e63abf929bf586de1375ebb7'
+            'dd2ff7e4ee3e6ef4ffc415a506ceb4a2f47976be8afda6bdb60a2830890e8b09c6c456b8ac67aa6aa8327bf0a11fb22039a80621e4cbbb145099b86ef87d7359')
 
 pkgver() {
-  cd "${srcdir}/mozc"
+  cd "${srcdir}/mozc" || exit
   source <(grep = src/data/version/mozc_version_template.bzl| tr -d ' ')
-  printf "%s.%s.%s.%s.%s" $MAJOR $MINOR $BUILD_OSS $((REVISION+2)) $_utdicdate
+  printf "%s.%s.%s.%s.%s" "$MAJOR" "$MINOR" "$BUILD_OSS" "$((REVISION+2))" "$_utdicdate"
 }
 
 prepare() {
-  # Enabling all dictionaries
-  sed -i '/^#.*="true"/ s/^#//' "${srcdir}/mozcdic-ut-${_utdicdate}/src/make-dictionaries.sh"
-
-  cd "$srcdir/mozc"
+  cd "$srcdir/mozc" || exit
   git submodule init
   git config submodule.src/third_party/abseil-cpp.url "$srcdir/abseil-cpp"
   git config submodule.src/third_party/breakpad.url "$srcdir/breakpad"
@@ -66,9 +86,9 @@ prepare() {
   git config submodule.src/third_party/protobuf.url "$srcdir/protobuf"
   git -c protocol.file.allow=always submodule update --init --recursive
 
-  cd src
+  cd src || exit
   # Generate zip code seed
-  msg "Generating zip code seed..."
+  echo "Generating zip code seed..."
   PYTHONPATH="$PWD:$PYTHONPATH" python dictionary/gen_zip_code_seed.py --zip_code="${srcdir}/x-ken-all.csv" --jigyosyo="${srcdir}/JIGYOSYO.CSV" >> data/dictionary_oss/dictionary09.txt
   echo "Done."
 
@@ -87,8 +107,22 @@ prepare() {
   # use libstdc++ instead of libc++
   sed '/stdlib=libc++/d;/-lc++/d' -i gyp/common.gypi
 
-  # Add UT dictionary
-  cat "${srcdir}/mozcdic-ut-${_utdicdate}/"mozcdic*-ut-*.txt >> data/dictionary_oss/dictionary00.txt
+  msg 'UT Dictionary steps, rewrite of `sh make.sh`'
+  cd "${srcdir}/merge-ut-dictionaries/src" || exit
+  msg '1. Append dictionaries'
+  for dict in "${_dict[@]}"; do
+    cat "$srcdir/mozcdic-ut-${dict}.txt" >> mozcdic-ut.txt
+  done
+  msg '2. Patch ruby scripts out from downloading'
+  sed '/^`wget*/d' -i count_word_hits.rb
+  sed "s,https://raw.githubusercontent.com/google/mozc/master,$srcdir/mozc," -i remove_duplicate_ut_entries.rb
+  cp -v "$srcdir/jawiki-latest-all-titles-in-ns0.gz" ./
+  msg '3. Run the ruby scripts as in original make.sh, it may take some time...'
+  ruby remove_duplicate_ut_entries.rb mozcdic-ut.txt
+  ruby count_word_hits.rb
+  ruby apply_word_hits.rb mozcdic-ut.txt
+  msg '4. Finally add UT dictionary to mozc source'
+  cat mozcdic-ut.txt >> "$srcdir/mozc/src/data/dictionary_oss/dictionary00.txt"
 }
 
 build() {
@@ -96,14 +130,14 @@ build() {
   CFLAGS="${CFLAGS} -fvisibility=hidden"
   CXXFLAGS="${CXXFLAGS} -fvisibility=hidden"
 
-  cd mozc/src
+  cd mozc/src || exit
 
   _targets="unix/fcitx5:fcitx5-mozc.so server:mozc_server gui/tool:mozc_tool renderer:mozc_renderer unix/ibus:ibus_mozc unix/emacs:mozc_emacs_helper"
   _options="-c opt --copt=-fPIC --config oss_linux"
 
   export JAVA_HOME='/usr/lib/jvm/java-11-openjdk/'
   export QT_BASE_PATH=/usr/include/qt
-  bazel build $_targets $_options
+  bazel build "$_targets" "$_options"
 
   # Extract license part of mozc
   head -n 29 server/mozc_server.cc > LICENSE
@@ -112,44 +146,44 @@ build() {
 package_mozc-ut-full-common() {
   pkgdesc="A Japanese Input Method for Chromium OS, Windows, Mac and Linux (the Open Source Edition of Google Japanese Input)"
   options=('!docs')
-  export PREFIX="${pkgdir}/usr"
+  export PREFIX="$pkgdir/usr"
   export _bldtype
-  cd mozc/src
+  cd mozc/src || exit
 
-  install -d "${pkgdir}/usr/share/licenses/${pkgname}/"
-  install -m 644 LICENSE data/installer/*.html "${pkgdir}/usr/share/licenses/${pkgname}/"
+  install -d "$pkgdir/usr/share/licenses/$pkgname/"
+  install -m 644 LICENSE data/installer/*.html "$pkgdir/usr/share/licenses/$pkgname/"
 
   ../scripts/install_server_bazel
 }
 
 package_fcitx5-mozc-ut-full() {
   pkgdesc="Fcitx5 module for Mozc with UT dictionary"
-  depends=('qt5-base' 'fcitx5' ${pkgbase}-common)
+  depends=('qt5-base' 'fcitx5' "$pkgbase-common")
   provides=('fcitx5-mozc')
   replaces=('fcitx5-mozc')
   conflicts=('fcitx-mozc' 'fcitx5-mozc')
 
-  export PREFIX="${pkgdir}/usr"
+  export PREFIX="$pkgdir/usr"
   export _bldtype
-  cd mozc/src
+  cd mozc/src || exit
 
   ../scripts/install_fcitx5_bazel
 }
 
 package_ibus-mozc-ut-full() {
   pkgdesc="IBus engine module for Mozc with UT dictionary"
-  depends=('ibus>=1.4.1' ${pkgbase}-common)
+  depends=('ibus>=1.4.1' "$pkgbase-common")
   replaces=('ibus-mozc')
   conflicts=('ibus-mozc')
 
   export _bldtype
-  cd "${srcdir}/mozc/src"
+  cd "${srcdir}/mozc/src" || exit
   install -D -m 755 bazel-bin/unix/ibus/ibus_mozc         "$pkgdir/usr/lib/ibus-mozc/ibus-engine-mozc"
   install -D -m 644 bazel-bin/unix/ibus/mozc.xml          "$pkgdir/usr/share/ibus/component/mozc.xml"
   install -D -m 755 bazel-bin/renderer/mozc_renderer      "${pkgdir}/usr/lib/mozc/mozc_renderer"
 
 
-  cd bazel-bin/unix
+  cd bazel-bin/unix || exit
   unzip -o icons.zip
 
   install -Dm644 mozc.png                                  "${pkgdir}/usr/share/ibus-mozc/product_icon.png"
@@ -164,15 +198,14 @@ package_ibus-mozc-ut-full() {
   install -Dm644 outlined/tool.svg                         "${pkgdir}/usr/share/ibus-mozc/tool.svg"
 }
 
-
 package_emacs-mozc-ut-full() {
   pkgdesc="Emacs engine module for Mozc with UT dictionary"
-  depends=('emacs' ${pkgbase}-common)
+  depends=('emacs' "$pkgbase-common")
   replaces=('emacs-mozc')
   conflicts=('emacs-mozc')
 
   export _bldtype
-  cd "${srcdir}/mozc/src"
+  cd "${srcdir}/mozc/src" || exit
   install -Dm755 bazel-bin/unix/emacs/mozc_emacs_helper "$pkgdir/usr/bin/mozc_emacs_helper"
   install -Dm644 unix/emacs/mozc.el                     "$pkgdir/usr/share/emacs/site-lisp/emacs-mozc/mozc.el"
 }