04.translate

#!/bin/bash
set -euo pipefail

LANGUAGE=$1
BATCH=$2
INPUT_FILE=$BATCH/sentences.gz
OUTPUT_FILE=$(dirname $INPUT_FILE)/sentences_${TARGET_LANG%~*}.gz
TMPSFX=${JOB_ID:-$$}

docs_src=$(gzip -dc $INPUT_FILE | wc -l)
lines_src=$(gzip -cd $INPUT_FILE | base64 -d | wc -l)

# Hint for translate scripts that want to guess a good batch size
export SIZE_HINT=$lines_src

# export BATCH for easy debugging down in the translation script
export BATCH TMPSFX

# Make sure these vars are at least available to the subprocess
gzip -cd $INPUT_FILE \
| b64filter cache $MODEL_DIR/${LANGUAGE}-${TARGET_LANG%~*}/translate.sh \
| gzip -9c > $OUTPUT_FILE.$TMPSFX

docs_dst=$(gzip -dc $OUTPUT_FILE.$TMPSFX | wc -l)
echo "Expecting $docs_src documents found $docs_dst"
test "$docs_src" -eq "$docs_dst" || exit 1

lines_dst=$(gzip -cd $OUTPUT_FILE.$TMPSFX | base64 -d | wc -l)
echo "Expecting $lines_src lines, found $lines_dst"
test "$lines_src" -eq "$lines_dst" || exit 1

mv $OUTPUT_FILE.$TMPSFX $OUTPUT_FILE