maint: rewrite translation management scripts

This commit is contained in:
Yury V. Zaytsev 2023-01-03 22:52:07 +01:00
parent b88d1b3790
commit 6da4346c4e
16 changed files with 189 additions and 418 deletions

View File

@ -0,0 +1,42 @@
# Translations maintenance
## Requirements
* tx
- Transifex client
- https://developers.transifex.com/docs/cli
* po4a
- Tool for converting translations between PO and other formats
- https://po4a.org
## Configuration
First time you run `tx` command it will ask you for your API token and create `~/.transifexrc`.
## Maintenance
Check the `*-from-transifex.py` (run by hand) and `*-to-transifex.py` (used in CI) scripts.
Wrapper for modern Transifex client:
```shell
#!/bin/sh
touch ~/.transifexrc
export XUID=$(id -u)
export XGID=$(id -g)
docker run \
--rm -i \
--user $XUID:$XGID \
--volume="/etc/group:/etc/group:ro" \
--volume="/etc/passwd:/etc/passwd:ro" \
--volume="/etc/shadow:/etc/shadow:ro" \
--volume $(pwd):/app \
--volume ~/.transifexrc:/.transifexrc \
--volume /etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt \
transifex/txcli \
--root-config /.transifexrc \
"$@"
```

View File

@ -1,29 +0,0 @@
== Pre-requirements ==
=== List of required apps ===
* tx (A transifex client. http://help.transifex.com/features/client/#user-client-08-install)
* po4a (A tool maintaining translations anywhere. http://alioth.debian.org/projects/po4a/)
=== An artifacts configuration ===
Put in the ~/.transifexrc file these lines:
[https://www.transifex.net]
hostname = https://www.transifex.net
username = YourTxLogin
password = YourTxPassword
token =
== Interact with Transifex via scripts ==
To get all translations from Transifex run:
find ./ -name '*-fromTransifex.*' -exec {} \;
To put source files to Transifex run:
find ./ -name '*-toTransifex.*' -exec {} \;

View File

@ -1,4 +1,6 @@
[po4a_langs] @translations@
[po4a_paths] var.d/$master/mc.doc.pot $lang:var.d/$master/$lang.po
[po4a_paths] var.d/$master/mc.hint.pot $lang:var.d/$master/$lang.po
[type:man] @srcdir@/doc/hints/$(docfile) $lang:@srcdir@/doc/hints/l10n/$(docfile).$lang
[options] --master-charset UTF-8 --localized-charset UTF-8 --keep 5 --wrap-po no
[type:asciidoc] @srcdir@/doc/hints/mc.hint $lang:@srcdir@/doc/hints/l10n/mc.hint.$lang

View File

@ -1,8 +1,7 @@
[main]
host = https://www.transifex.com
[mc.mc_hint]
[o:mc:p:mc:r:mc-hint--master]
file_filter = <lang>.po
source_file = mc.doc.pot
source_file = mc.hint.pot
source_lang = en

View File

@ -1,7 +1,7 @@
[main]
host = https://www.transifex.com
[mc.mcpot]
[o:mc:p:mc:r:mc-pot--master]
file_filter = <lang>.po
source_file = mc.pot
source_lang = en

View File

@ -1,161 +0,0 @@
#!/bin/bash
# Midnight Commander - common functions for playing with translations.
#
# Copyright (C) 2013
# The Free Software Foundation, Inc.
#
# Written by:
# Slava Zanko <slavazanko@gmail.com>, 2013
#
# This file is part of the Midnight Commander.
#
# The Midnight Commander is free software: you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation, either version 3 of the License,
# or (at your option) any later version.
#
# The Midnight Commander is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#*** include section (source functions, for example) *******************
#*** file scope functions **********************************************
# ----------------------------------------------------------------------
## Returns name of working directory.
##
## @param sync_file_name file name which will be sync'ed with Transifex
## @return name of working directory
getSyncDirName() {
sync_file_name=$1; shift
echo "${SYNC_TX_CURRENT_DIR}/var.d/${sync_file_name}"
}
# ----------------------------------------------------------------------
## Returns config file name as full path.
##
## @param sync_file_name file name which will be sync'ed with Transifex
## @param config_file_name config file name
## @return config file name
getConfigFile() {
sync_file_name=$1; shift
config_file_name=$1; shift
echo "${MC_SOURCE_ROOT_DIR}/maint/utils/sync-transifex/config.d/${sync_file_name}/${config_file_name}"
}
# ----------------------------------------------------------------------
## Returns name of working directory. Init the directory, if needed.
##
## @param sync_file_name file name which will be sync'ed with Transifex
## @return name of working directory
initSyncDirIfNeeded() {
sync_file_name=$1; shift
sync_dir_name=$(getSyncDirName "${sync_file_name}")
[ ! -e "${sync_dir_name}" ] && mkdir -p "${sync_dir_name}/.tx"
cp \
"$(getConfigFile ${sync_file_name} 'tx.config')" \
"${sync_dir_name}/.tx/config"
echo "${sync_dir_name}"
}
# ----------------------------------------------------------------------
## Convert file from plain text to POT-file format.
##
## @param source_file path to plain text file
## @param destination_file path to po-file
convertFromTextToPo() {
source_file=$1; shift
destination_file=$1; shift
po4a-gettextize -f text -m "${source_file}" -p "${destination_file}"
}
# ----------------------------------------------------------------------
## Send POT-file to Transifex.
##
## @param tx_work_dir working directory where placed the POT file and
## a config-file for tx utility
sendSourceToTransifex() {
tx_work_dir=$1; shift
tx -r "${tx_work_dir}" push -s
}
# ----------------------------------------------------------------------
## Reveive translations from Transifex.
##
## @param tx_work_dir working directory where placed the POT file and
## a config-file for tx utility
receiveTranslationsFromTransifex() {
tx_work_dir=$1; shift
pushd "${tx_work_dir}" >/dev/null
tx pull --all --force
popd >/dev/null
}
# ----------------------------------------------------------------------
## Create po4a.cfg file.
##
## @param sync_file_name file name which will be sync'ed with Transifex
createPo4A() {
sync_file_name=$1; shift
sync_dir_name=$(getSyncDirName "${sync_file_name}")
[ ! -e "${sync_dir_name}" ] && mkdir -p "${sync_dir_name}"
pushd $sync_dir_name >/dev/null
translations=$(ls *.po| sed -e 's/.po//g'| tr '\n' ' ')
popd >/dev/null
sed -e 's!@srcdir@!'"${MC_SOURCE_ROOT_DIR}"'!g' \
-e 's!@translations@!'"${translations}"'!g' \
"$(getConfigFile ${sync_file_name} 'po4a.cfg')" \
> "${sync_dir_name}/po4a.cfg"
}
# ----------------------------------------------------------------------
## Create po4a.cfg file.
##
## @param sync_dir_name working directory where placed translations
## @param sync_file_name file name which will be sync'ed with Transifex
convertFromPoToText() {
sync_dir_name=$1; shift
sync_file_name=$1; shift
pushd "${SYNC_TX_CURRENT_DIR}" >/dev/null
po4a -k 0 -q \
-M UTF-8 -L UTF-8 \
--variable docfile="${sync_file_name}" \
"${sync_dir_name}/po4a.cfg"
popd >/dev/null
}
# ----------------------------------------------------------------------
#*** main code *********************************************************
SYNC_TX_CURRENT_DIR=${SYNC_TX_CURRENT_DIR:-${MC_SOURCE_ROOT_DIR}/maint/utils/sync-transifex}

View File

@ -0,0 +1,31 @@
#!/usr/bin/env python3
import glob
import subprocess
from pathlib import Path
from textwrap import wrap
from translation_utils import create_po4a_config, init_sync_dir
RESOURCE_NAME = "mc.hint"
SCRIPT_DIR = Path(__file__).parent
SOURCE_DIR = SCRIPT_DIR.parent.parent.parent
def unwrap_paragraphs():
hint_files = glob.glob(str(SOURCE_DIR / "doc" / "hints" / "l10n" / "mc.hint.*"))
for hint_file in map(Path, hint_files):
lines = hint_file.read_text().split("\n\n")
hint_file.write_text("\n\n".join("".join(wrap(line, width=1024)) for line in lines) + "\n")
sync_dir = init_sync_dir(SCRIPT_DIR, RESOURCE_NAME)
subprocess.run(("tx", "pull", "--all", "--force"), cwd=sync_dir, check=True)
po4a_config = create_po4a_config(sync_dir, SCRIPT_DIR, SOURCE_DIR, RESOURCE_NAME)
subprocess.run(("po4a", str(po4a_config)), cwd=SCRIPT_DIR, check=True)
unwrap_paragraphs()

View File

@ -0,0 +1,19 @@
#!/usr/bin/env python3
import subprocess
from pathlib import Path
from translation_utils import create_po4a_config, init_sync_dir
RESOURCE_NAME = "mc.hint"
SCRIPT_DIR = Path(__file__).parent
SOURCE_DIR = SCRIPT_DIR.parent.parent.parent
sync_dir = init_sync_dir(SCRIPT_DIR, RESOURCE_NAME)
po4a_config = create_po4a_config(sync_dir, SCRIPT_DIR, SOURCE_DIR, RESOURCE_NAME)
subprocess.run(("po4a", str(po4a_config)), cwd=SCRIPT_DIR, check=True)
subprocess.run(("tx", "push", "--source"), cwd=sync_dir, check=True)

View File

@ -1,61 +0,0 @@
#!/bin/bash
# Midnight Commander - fetch doc/hints/mc.hint translations from Transifex
#
# Copyright (C) 2013
# The Free Software Foundation, Inc.
#
# Written by:
# Slava Zanko <slavazanko@gmail.com>, 2013
#
# This file is part of the Midnight Commander.
#
# The Midnight Commander is free software: you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation, either version 3 of the License,
# or (at your option) any later version.
#
# The Midnight Commander is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
set -e
MC_SOURCE_ROOT_DIR=${MC_SOURCE_ROOT_DIR:-$(dirname $(dirname $(dirname $(pwd))))}
#*** include section (source functions, for example) *******************
source "${MC_SOURCE_ROOT_DIR}/maint/utils/sync-transifex/functions"
#*** file scope functions **********************************************
processHintFiles() {
# Remove extra backslash
sed -i -e 's/\\-/-/g' ${MC_SOURCE_ROOT_DIR}/doc/hints/l10n/mc.hint.*
# Remove extra line breaks
for fn in ${MC_SOURCE_ROOT_DIR}/doc/hints/l10n/mc.hint.*; do
awk '/^$/ { print "\n"; } /./ { printf("%s ", $0); } END { print; }' $fn > $fn.tmp
sed -e 's/[[:space:]]*$//' < $fn.tmp > $fn
perl -i -0pe 's/\n+\Z/\n/' $fn
rm $fn.tmp
done
}
#*** main code *********************************************************
WORK_DIR=$(initSyncDirIfNeeded "mc.hint")
receiveTranslationsFromTransifex "${WORK_DIR}"
createPo4A "mc.hint"
convertFromPoToText "${WORK_DIR}" "mc.hint"
processHintFiles

View File

@ -1,43 +0,0 @@
#!/bin/bash
# Midnight Commander - push doc/hints/mc.hint file to Transifex
#
# Copyright (C) 2013
# The Free Software Foundation, Inc.
#
# Written by:
# Slava Zanko <slavazanko@gmail.com>, 2013
#
# This file is part of the Midnight Commander.
#
# The Midnight Commander is free software: you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation, either version 3 of the License,
# or (at your option) any later version.
#
# The Midnight Commander is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
set -e
MC_SOURCE_ROOT_DIR=${MC_SOURCE_ROOT_DIR:-$(dirname $(dirname $(dirname $(pwd))))}
#*** include section (source functions, for example) *******************
source "${MC_SOURCE_ROOT_DIR}/maint/utils/sync-transifex/functions"
#*** file scope functions **********************************************
#*** main code *********************************************************
WORK_DIR=$(initSyncDirIfNeeded "mc.hint")
convertFromTextToPo "${MC_SOURCE_ROOT_DIR}/doc/hints/mc.hint" "${WORK_DIR}/mc.doc.pot"
sendSourceToTransifex "${WORK_DIR}"

View File

@ -0,0 +1,41 @@
#!/usr/bin/env python3
import glob
import subprocess
from pathlib import Path
from translation_utils import get_translations, init_sync_dir
RESOURCE_NAME = "mc.pot"
SCRIPT_DIR = Path(__file__).parent
SOURCE_DIR = SCRIPT_DIR.parent.parent.parent
PO_DIR = SOURCE_DIR / "po"
def strip_message_locations(work_dir: Path):
for po_file in (work_dir / filename for filename in glob.glob("*.po", root_dir=work_dir)):
po_file.write_text(
"".join(line for line in po_file.read_text().splitlines(keepends=True) if not line.startswith("#:"))
)
def copy_translations_to_source_dir(source_dir: Path, target_dir: Path):
for po_file in (source_dir / filename for filename in glob.glob("*.po", root_dir=source_dir)):
(target_dir / po_file.name).write_text(po_file.read_text())
def update_linguas(po_dir: Path):
translations = get_translations(po_dir)
(po_dir / "LINGUAS").write_text("# List of available translations\n" + "\n".join(translations) + "\n")
sync_dir = init_sync_dir(SCRIPT_DIR, RESOURCE_NAME)
subprocess.run(("tx", "pull", "--all", "--force"), cwd=sync_dir, check=True)
strip_message_locations(sync_dir)
copy_translations_to_source_dir(sync_dir, PO_DIR)
update_linguas(PO_DIR)

View File

@ -1,67 +0,0 @@
#!/bin/bash
# Midnight Commander - fetch doc/hints/mc.hint translations from Transifex
#
# Copyright (C) 2013
# The Free Software Foundation, Inc.
#
# Written by:
# Slava Zanko <slavazanko@gmail.com>, 2013
#
# This file is part of the Midnight Commander.
#
# The Midnight Commander is free software: you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation, either version 3 of the License,
# or (at your option) any later version.
#
# The Midnight Commander is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
set -e
MC_SOURCE_ROOT_DIR=${MC_SOURCE_ROOT_DIR:-$(dirname $(dirname $(dirname $(pwd))))}
#*** include section (source functions, for example) *******************
source "${MC_SOURCE_ROOT_DIR}/maint/utils/sync-transifex/functions"
#*** file scope functions **********************************************
stripLocation() {
work_dir=$1; shift
for i in $(find "${work_dir}" -name '*.po' -print); do
sed -i '/^#:/d' "${i}"
done
}
# ----------------------------------------------------------------------
copyFilesToSourceDir() {
work_dir=$1; shift
source_dir=$1; shift
exclude_list_file=$(getConfigFile "po" "po-ignore.list")
for i in $(find "${work_dir}" -name '*.po' -print | sort); do
[ $(grep -c "^\s*$(basename ${i})" "${exclude_list_file}") -ne 1 ] && {
cp -f "${i}" "${source_dir}"
}
done
}
#*** main code *********************************************************
WORK_DIR=$(initSyncDirIfNeeded "po")
receiveTranslationsFromTransifex "${WORK_DIR}"
stripLocation "${WORK_DIR}"
copyFilesToSourceDir "${WORK_DIR}" "${MC_SOURCE_ROOT_DIR}/po"

View File

@ -0,0 +1,18 @@
#!/usr/bin/env python3
import subprocess
from pathlib import Path
from translation_utils import init_sync_dir
RESOURCE_NAME = "mc.pot"
SCRIPT_DIR = Path(__file__).parent
SOURCE_DIR = SCRIPT_DIR.parent.parent.parent
sync_dir = init_sync_dir(SCRIPT_DIR, RESOURCE_NAME)
# Copy mc.pot to the working directory
(sync_dir / RESOURCE_NAME).write_text((SOURCE_DIR / "po" / RESOURCE_NAME).read_text())
subprocess.run(("tx", "push", "--source"), cwd=sync_dir, check=True)

View File

@ -1,50 +0,0 @@
#!/bin/bash
# Midnight Commander - push doc/hints/mc.hint file to Transifex
#
# Copyright (C) 2013
# The Free Software Foundation, Inc.
#
# Written by:
# Slava Zanko <slavazanko@gmail.com>, 2013
#
# This file is part of the Midnight Commander.
#
# The Midnight Commander is free software: you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation, either version 3 of the License,
# or (at your option) any later version.
#
# The Midnight Commander is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
set -e
MC_SOURCE_ROOT_DIR=${MC_SOURCE_ROOT_DIR:-$(dirname $(dirname $(dirname $(pwd))))}
#*** include section (source functions, for example) *******************
source "${MC_SOURCE_ROOT_DIR}/maint/utils/sync-transifex/functions"
#*** file scope functions **********************************************
copyPotToWorkDir() {
work_dir=$1; shift
source_dir=$1; shift
cp -f "${source_dir}/mc.pot" "${work_dir}"
}
#*** main code *********************************************************
WORK_DIR=$(initSyncDirIfNeeded "po")
copyPotToWorkDir "${WORK_DIR}" "${MC_SOURCE_ROOT_DIR}/po"
sendSourceToTransifex "${WORK_DIR}"

View File

@ -0,0 +1,31 @@
import glob
from pathlib import Path
def get_config_file(root_dir: Path, resource: str, name: str) -> Path:
return root_dir / "config.d" / resource / name
def init_sync_dir(root_dir: Path, resource: str) -> Path:
tx_dir = root_dir / "var.d" / resource / ".tx"
tx_dir.mkdir(parents=True, exist_ok=True)
(tx_dir / "config").write_text(get_config_file(root_dir, resource, "tx.config").read_text())
return tx_dir.parent
def create_po4a_config(sync_dir: Path, script_dir: Path, source_dir: Path, resource: str) -> Path:
translations = get_translations(sync_dir)
config = get_config_file(script_dir, resource, "po4a.cfg").read_text()
config = config.replace("@srcdir@", str(source_dir))
config = config.replace("@translations@", " ".join(translations))
config_path = sync_dir / "po4a.cfg"
config_path.write_text(config)
return config_path
def get_translations(root_dir: Path) -> list[str]:
return sorted(Path(filename).name.removesuffix(".po") for filename in glob.glob("*.po", root_dir=root_dir))