Update pten and move to prod (#39)

* Update pten and move to prod

* Update model registry [skip ci]

* Fix ci workflow

* Update evaluation results [skip ci]

Co-authored-by: CircleCI evaluation job <ci-models-evaluation@firefox-translations>

.circleci/config.yml

@@ -102,12 +102,6 @@ workflows:
   version: 2
   ci:
     jobs:
-      - deploy:
-          filters:
-            branches:
-              ignore: /.*/
-            tags:
-              only: /\d*\.\d*\.\d*/
       - evaluate:
           filters:
             branches:
@@ -116,9 +110,19 @@ workflows:
                 - /pull\/[0-9]+/
                 - main
       - generate-registry:
+          requires:
+            - evaluate
           filters:
             branches:
               # Forked pull requests have CIRCLE_BRANCH set to pull/XXX
               ignore:
                 - /pull\/[0-9]+/
                 - main
+  publish:
+    jobs:
+      - deploy:
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /\d*\.\d*\.\d*/

evaluation/dev/pt-en/flores-dev.bergamot.en.bleu

@@ -1 +0,0 @@
-45.9

evaluation/dev/pt-en/flores-dev.google.en.bleu

@@ -1 +0,0 @@
-49.5

evaluation/dev/pt-en/flores-test.bergamot.en.bleu

@@ -1 +0,0 @@
-44.7

evaluation/dev/pt-en/flores-test.google.en.bleu

@@ -1 +0,0 @@
-48.9

evaluation/dev/pt-en/mtedx_test.bergamot.en.bleu

@@ -1 +0,0 @@
-39.7

evaluation/dev/pt-en/mtedx_test.google.en.bleu

@@ -1 +0,0 @@
-38.9

evaluation/dev/pt-en/mtedx_test.microsoft.en.bleu

@@ -1 +0,0 @@
-40.9

evaluation/dev/results.md

@@ -57,84 +57,24 @@ Both absolute and relative differences in BLEU scores between Bergamot and other
 
 ## avg
 
-| Translator/Dataset | fa-en | pt-en | ru-en | it-en | en-ru | en-it | en-fa | en-pt | is-en |
-| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| bergamot | 28.70 | 43.43 | 33.37 | 32.30 | 30.47 | 29.53 | 17.30 | 46.55 | 23.50 |
-| google | 36.05 (+7.35, +25.61%) | 45.77 (+2.33, +5.37%) | 36.53 (+3.15, +9.45%) | 34.50 (+2.20, +6.81%) | 33.72 (+3.25, +10.67%) | 29.47 (-0.07, -0.23%) | 27.70 (+10.40, +60.12%) | 56.05 (+9.50, +20.41%) | 34.95 (+11.45, +48.72%) |
-| microsoft | 36.15 (+7.45, +25.96%) | 46.43 (+3.00, +6.91%) | 36.87 (+3.50, +10.49%) | 34.55 (+2.25, +6.97%) | 33.68 (+3.21, +10.53%) | 32.20 (+2.67, +9.03%) | 20.50 (+3.20, +18.50%) | 50.25 (+3.70, +7.95%) | 34.90 (+11.40, +48.51%) |
+| Translator/Dataset | en-it | en-pt | ru-en | en-ru | en-fa | fa-en | is-en | it-en |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| bergamot | 29.53 | 46.55 | 33.37 | 30.47 | 17.30 | 28.70 | 23.50 | 32.30 |
+| google | 29.47 (-0.07, -0.23%) | 56.05 (+9.50, +20.41%) | 36.53 (+3.15, +9.45%) | 33.72 (+3.25, +10.67%) | 27.70 (+10.40, +60.12%) | 36.05 (+7.35, +25.61%) | 34.95 (+11.45, +48.72%) | 34.50 (+2.20, +6.81%) |
+| microsoft | 32.20 (+2.67, +9.03%) | 50.25 (+3.70, +7.95%) | 36.87 (+3.50, +10.49%) | 33.68 (+3.21, +10.53%) | 20.50 (+3.20, +18.50%) | 36.15 (+7.45, +25.96%) | 34.90 (+11.40, +48.51%) | 34.55 (+2.25, +6.97%) |
 
 ![Results](img/avg.png)
 
-## fa-en
-
-| Translator/Dataset | flores-dev | flores-test |
-| --- | --- | --- |
-| bergamot | 29.10 | 28.30 |
-| google | 36.70 (+7.60, +26.12%) | 35.40 (+7.10, +25.09%) |
-| microsoft | 36.50 (+7.40, +25.43%) | 35.80 (+7.50, +26.50%) |
-
-![Results](img/fa-en.png)
-
-## pt-en
-
-| Translator/Dataset | mtedx_test | flores-dev | flores-test |
-| --- | --- | --- | --- |
-| bergamot | 39.70 | 45.90 | 44.70 |
-| google | 38.90 (-0.80, -2.02%) | 49.50 (+3.60, +7.84%) | 48.90 (+4.20, +9.40%) |
-| microsoft | 40.90 (+1.20, +3.02%) | 49.80 (+3.90, +8.50%) | 48.60 (+3.90, +8.72%) |
-
-![Results](img/pt-en.png)
-
-## ru-en
-
-| Translator/Dataset | wmt17 | wmt18 | wmt14 | mtedx_test | wmt16 | wmt13 | wmt19 | wmt20 | flores-dev | flores-test | wmt15 |
-| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| bergamot | 37.90 | 32.30 | 38.00 | 24.20 | 33.40 | 29.50 | 39.30 | 35.40 | 31.90 | 31.50 | 33.70 |
-| google | 41.50 (+3.60, +9.50%) | 36.00 (+3.70, +11.46%) | 41.20 (+3.20, +8.42%) | 25.00 (+0.80, +3.31%) | 36.60 (+3.20, +9.58%) | 31.40 (+1.90, +6.44%) | 42.40 (+3.10, +7.89%) | 37.70 (+2.30, +6.50%) | 37.00 (+5.10, +15.99%) | 35.50 (+4.00, +12.70%) | 37.50 (+3.80, +11.28%) |
-| microsoft | 41.60 (+3.70, +9.76%) | 36.90 (+4.60, +14.24%) | 41.70 (+3.70, +9.74%) | 26.10 (+1.90, +7.85%) | 37.60 (+4.20, +12.57%) | 31.20 (+1.70, +5.76%) | 42.60 (+3.30, +8.40%) | 37.80 (+2.40, +6.78%) | 36.20 (+4.30, +13.48%) | 36.10 (+4.60, +14.60%) | 37.80 (+4.10, +12.17%) |
-
-![Results](img/ru-en.png)
-
-## it-en
-
-| Translator/Dataset | wmt09 | mtedx_test | flores-dev | flores-test |
-| --- | --- | --- | --- | --- |
-| bergamot | 33.10 | 35.50 | 30.70 | 29.90 |
-| google | 35.40 (+2.30, +6.95%) | 35.90 (+0.40, +1.13%) | 33.40 (+2.70, +8.79%) | 33.30 (+3.40, +11.37%) |
-| microsoft | 35.80 (+2.70, +8.16%) | 36.40 (+0.90, +2.54%) | 33.30 (+2.60, +8.47%) | 32.70 (+2.80, +9.36%) |
-
-![Results](img/it-en.png)
-
-## en-ru
-
-| Translator/Dataset | wmt15 | flores-dev | wmt13 | wmt16 | wmt20 | flores-test | wmt19 | wmt17 | wmt18 | wmt14 |
-| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| bergamot | 31.80 | 30.20 | 26.70 | 31.60 | 22.00 | 29.10 | 31.70 | 34.10 | 29.00 | 38.50 |
-| google | 35.00 (+3.20, +10.06%) | 33.60 (+3.40, +11.26%) | 27.60 (+0.90, +3.37%) | 34.00 (+2.40, +7.59%) | 26.40 (+4.40, +20.00%) | 33.60 (+4.50, +15.46%) | 32.20 (+0.50, +1.58%) | 37.30 (+3.20, +9.38%) | 34.10 (+5.10, +17.59%) | 43.40 (+4.90, +12.73%) |
-| microsoft | 35.60 (+3.80, +11.95%) | 33.20 (+3.00, +9.93%) | 27.10 (+0.40, +1.50%) | 33.80 (+2.20, +6.96%) | 26.00 (+4.00, +18.18%) | 33.00 (+3.90, +13.40%) | 32.70 (+1.00, +3.15%) | 38.10 (+4.00, +11.73%) | 33.20 (+4.20, +14.48%) | 44.10 (+5.60, +14.55%) |
-
-![Results](img/en-ru.png)
-
 ## en-it
 
-| Translator/Dataset | flores-dev | flores-test | wmt09 |
+| Translator/Dataset | flores-test | flores-dev | wmt09 |
 | --- | --- | --- | --- |
-| bergamot | 28.40 | 29.20 | 31.00 |
-| google | 29.00 (+0.60, +2.11%) | 30.10 (+0.90, +3.08%) | 29.30 (-1.70, -5.48%) |
-| microsoft | 31.00 (+2.60, +9.15%) | 32.00 (+2.80, +9.59%) | 33.60 (+2.60, +8.39%) |
+| bergamot | 29.20 | 28.40 | 31.00 |
+| google | 30.10 (+0.90, +3.08%) | 29.00 (+0.60, +2.11%) | 29.30 (-1.70, -5.48%) |
+| microsoft | 32.00 (+2.80, +9.59%) | 31.00 (+2.60, +9.15%) | 33.60 (+2.60, +8.39%) |
 
 ![Results](img/en-it.png)
 
-## en-fa
-
-| Translator/Dataset | flores-dev | flores-test |
-| --- | --- | --- |
-| bergamot | 17.20 | 17.40 |
-| google | 27.20 (+10.00, +58.14%) | 28.20 (+10.80, +62.07%) |
-| microsoft | 19.90 (+2.70, +15.70%) | 21.10 (+3.70, +21.26%) |
-
-![Results](img/en-fa.png)
-
 ## en-pt
 
 | Translator/Dataset | flores-test | flores-dev |
@@ -145,6 +85,46 @@ Both absolute and relative differences in BLEU scores between Bergamot and other
 
 ![Results](img/en-pt.png)
 
+## ru-en
+
+| Translator/Dataset | mtedx_test | wmt19 | wmt17 | flores-dev | flores-test | wmt14 | wmt15 | wmt16 | wmt13 | wmt18 | wmt20 |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| bergamot | 24.20 | 39.30 | 37.90 | 31.90 | 31.50 | 38.00 | 33.70 | 33.40 | 29.50 | 32.30 | 35.40 |
+| google | 25.00 (+0.80, +3.31%) | 42.40 (+3.10, +7.89%) | 41.50 (+3.60, +9.50%) | 37.00 (+5.10, +15.99%) | 35.50 (+4.00, +12.70%) | 41.20 (+3.20, +8.42%) | 37.50 (+3.80, +11.28%) | 36.60 (+3.20, +9.58%) | 31.40 (+1.90, +6.44%) | 36.00 (+3.70, +11.46%) | 37.70 (+2.30, +6.50%) |
+| microsoft | 26.10 (+1.90, +7.85%) | 42.60 (+3.30, +8.40%) | 41.60 (+3.70, +9.76%) | 36.20 (+4.30, +13.48%) | 36.10 (+4.60, +14.60%) | 41.70 (+3.70, +9.74%) | 37.80 (+4.10, +12.17%) | 37.60 (+4.20, +12.57%) | 31.20 (+1.70, +5.76%) | 36.90 (+4.60, +14.24%) | 37.80 (+2.40, +6.78%) |
+
+![Results](img/ru-en.png)
+
+## en-ru
+
+| Translator/Dataset | wmt16 | wmt15 | flores-dev | wmt18 | wmt14 | wmt17 | wmt20 | wmt13 | wmt19 | flores-test |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| bergamot | 31.60 | 31.80 | 30.20 | 29.00 | 38.50 | 34.10 | 22.00 | 26.70 | 31.70 | 29.10 |
+| google | 34.00 (+2.40, +7.59%) | 35.00 (+3.20, +10.06%) | 33.60 (+3.40, +11.26%) | 34.10 (+5.10, +17.59%) | 43.40 (+4.90, +12.73%) | 37.30 (+3.20, +9.38%) | 26.40 (+4.40, +20.00%) | 27.60 (+0.90, +3.37%) | 32.20 (+0.50, +1.58%) | 33.60 (+4.50, +15.46%) |
+| microsoft | 33.80 (+2.20, +6.96%) | 35.60 (+3.80, +11.95%) | 33.20 (+3.00, +9.93%) | 33.20 (+4.20, +14.48%) | 44.10 (+5.60, +14.55%) | 38.10 (+4.00, +11.73%) | 26.00 (+4.00, +18.18%) | 27.10 (+0.40, +1.50%) | 32.70 (+1.00, +3.15%) | 33.00 (+3.90, +13.40%) |
+
+![Results](img/en-ru.png)
+
+## en-fa
+
+| Translator/Dataset | flores-test | flores-dev |
+| --- | --- | --- |
+| bergamot | 17.40 | 17.20 |
+| google | 28.20 (+10.80, +62.07%) | 27.20 (+10.00, +58.14%) |
+| microsoft | 21.10 (+3.70, +21.26%) | 19.90 (+2.70, +15.70%) |
+
+![Results](img/en-fa.png)
+
+## fa-en
+
+| Translator/Dataset | flores-dev | flores-test |
+| --- | --- | --- |
+| bergamot | 29.10 | 28.30 |
+| google | 36.70 (+7.60, +26.12%) | 35.40 (+7.10, +25.09%) |
+| microsoft | 36.50 (+7.40, +25.43%) | 35.80 (+7.50, +26.50%) |
+
+![Results](img/fa-en.png)
+
 ## is-en
 
 | Translator/Dataset | flores-dev | flores-test |
@@ -153,4 +133,14 @@ Both absolute and relative differences in BLEU scores between Bergamot and other
 | google | 35.50 (+11.90, +50.42%) | 34.40 (+11.00, +47.01%) |
 | microsoft | 35.10 (+11.50, +48.73%) | 34.70 (+11.30, +48.29%) |
 
-![Results](img/is-en.png)
+![Results](img/is-en.png)
+
+## it-en
+
+| Translator/Dataset | mtedx_test | flores-dev | flores-test | wmt09 |
+| --- | --- | --- | --- | --- |
+| bergamot | 35.50 | 30.70 | 29.90 | 33.10 |
+| google | 35.90 (+0.40, +1.13%) | 33.40 (+2.70, +8.79%) | 33.30 (+3.40, +11.37%) | 35.40 (+2.30, +6.95%) |
+| microsoft | 36.40 (+0.90, +2.54%) | 33.30 (+2.60, +8.47%) | 32.70 (+2.80, +9.36%) | 35.80 (+2.70, +8.16%) |
+
+![Results](img/it-en.png)

evaluation/prod/pt-en/flores-dev.bergamot.en.bleu

@@ -0,0 +1 @@
+47.8

evaluation/prod/pt-en/flores-dev.google.en.bleu

@@ -0,0 +1 @@
+50.4

evaluation/prod/pt-en/flores-test.bergamot.en.bleu

@@ -0,0 +1 @@
+46.6

evaluation/prod/pt-en/flores-test.google.en.bleu

@@ -0,0 +1 @@
+50.3

evaluation/prod/pt-en/mtedx_test.bergamot.en.bleu

@@ -0,0 +1 @@
+40.2

evaluation/prod/pt-en/mtedx_test.google.en.bleu

@@ -0,0 +1 @@
+39.1

evaluation/prod/pt-en/mtedx_test.microsoft.en.bleu

@@ -0,0 +1 @@
+41.0

evaluation/prod/results.md

@@ -57,23 +57,43 @@ Both absolute and relative differences in BLEU scores between Bergamot and other
 
 ## avg
 
-| Translator/Dataset | es-en | nb-en | bg-en | et-en | en-cs | cs-en | de-en | en-es | en-et | en-bg | en-de |
-| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| bergamot | 32.38 | 37.60 | 38.50 | 32.37 | 24.65 | 30.34 | 33.51 | 32.41 | 25.50 | 42.10 | 32.27 |
-| google | 33.64 (+1.27, +3.91%) | 42.05 (+4.45, +11.84%) | 41.30 (+2.80, +7.27%) | 35.80 (+3.43, +10.61%) | 26.73 (+2.09, +8.47%) | 32.40 (+2.06, +6.80%) | 35.98 (+2.48, +7.39%) | 34.74 (+2.32, +7.17%) | 28.60 (+3.10, +12.16%) | 44.60 (+2.50, +5.94%) | 33.05 (+0.77, +2.40%) |
-| microsoft | 32.93 (+0.56, +1.72%) | 42.90 (+5.30, +14.10%) | 41.20 (+2.70, +7.01%) | 36.17 (+3.80, +11.74%) | 27.75 (+3.11, +12.60%) | 33.53 (+3.19, +10.53%) | 38.21 (+4.70, +14.03%) | 33.76 (+1.35, +4.17%) | 28.47 (+2.97, +11.63%) | 38.55 (-3.55, -8.43%) | 33.54 (+1.27, +3.93%) |
+| Translator/Dataset | cs-en | en-et | et-en | nb-en | bg-en | en-es | en-bg | en-cs | de-en | pt-en | es-en | en-de |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| bergamot | 30.34 | 25.50 | 32.37 | 37.60 | 38.50 | 32.41 | 42.10 | 24.65 | 33.51 | 44.87 | 32.38 | 32.27 |
+| google | 32.40 (+2.06, +6.80%) | 28.60 (+3.10, +12.16%) | 35.80 (+3.43, +10.61%) | 42.05 (+4.45, +11.84%) | 41.30 (+2.80, +7.27%) | 34.74 (+2.32, +7.17%) | 44.60 (+2.50, +5.94%) | 26.73 (+2.09, +8.47%) | 35.98 (+2.48, +7.39%) | 46.60 (+1.73, +3.86%) | 33.64 (+1.27, +3.91%) | 33.05 (+0.77, +2.40%) |
+| microsoft | 33.53 (+3.19, +10.53%) | 28.47 (+2.97, +11.63%) | 36.17 (+3.80, +11.74%) | 42.90 (+5.30, +14.10%) | 41.20 (+2.70, +7.01%) | 33.76 (+1.35, +4.17%) | 38.55 (-3.55, -8.43%) | 27.75 (+3.11, +12.60%) | 38.21 (+4.70, +14.03%) | 46.47 (+1.60, +3.57%) | 32.93 (+0.56, +1.72%) | 33.54 (+1.27, +3.93%) |
 
 ![Results](img/avg.png)
 
-## es-en
+## cs-en
 
-| Translator/Dataset | wmt10 | wmt09 | wmt11 | mtedx_test | wmt13 | wmt08 | flores-dev | flores-test | wmt12 |
-| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| bergamot | 35.80 | 29.40 | 34.30 | 36.80 | 35.20 | 27.30 | 27.50 | 26.80 | 38.30 |
-| google | 37.00 (+1.20, +3.35%) | 31.60 (+2.20, +7.48%) | 35.20 (+0.90, +2.62%) | 35.40 (-1.40, -3.80%) | 35.70 (+0.50, +1.42%) | 28.30 (+1.00, +3.66%) | 30.50 (+3.00, +10.91%) | 30.30 (+3.50, +13.06%) | 38.80 (+0.50, +1.31%) |
-| microsoft | 35.40 (-0.40, -1.12%) | 29.60 (+0.20, +0.68%) | 33.70 (-0.60, -1.75%) | 37.60 (+0.80, +2.17%) | 35.30 (+0.10, +0.28%) | 26.80 (-0.50, -1.83%) | 30.30 (+2.80, +10.18%) | 29.90 (+3.10, +11.57%) | 37.80 (-0.50, -1.31%) |
+| Translator/Dataset | wmt08 | wmt17 | wmt10 | flores-dev | flores-test | wmt12 | wmt11 | wmt14 | wmt15 | wmt16 | wmt13 | wmt18 | wmt09 | wmt20 |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| bergamot | 24.50 | 30.20 | 28.20 | 35.30 | 35.30 | 26.50 | 28.10 | 35.00 | 32.00 | 33.40 | 30.30 | 31.30 | 27.60 | 27.00 |
+| google | 26.30 (+1.80, +7.35%) | 31.20 (+1.00, +3.31%) | 30.50 (+2.30, +8.16%) | 38.60 (+3.30, +9.35%) | 39.00 (+3.70, +10.48%) | 28.60 (+2.10, +7.92%) | 30.20 (+2.10, +7.47%) | 38.00 (+3.00, +8.57%) | 33.60 (+1.60, +5.00%) | 34.80 (+1.40, +4.19%) | 32.40 (+2.10, +6.93%) | 32.10 (+0.80, +2.56%) | 29.90 (+2.30, +8.33%) | 28.40 (+1.40, +5.19%) |
+| microsoft | 26.40 (+1.90, +7.76%) | 33.60 (+3.40, +11.26%) | 30.70 (+2.50, +8.87%) | 40.00 (+4.70, +13.31%) | 40.30 (+5.00, +14.16%) | 29.70 (+3.20, +12.08%) | 30.90 (+2.80, +9.96%) | 39.90 (+4.90, +14.00%) | 34.70 (+2.70, +8.44%) | 38.30 (+4.90, +14.67%) | 33.40 (+3.10, +10.23%) | 34.30 (+3.00, +9.58%) | 29.60 (+2.00, +7.25%) | 27.60 (+0.60, +2.22%) |
 
-![Results](img/es-en.png)
+![Results](img/cs-en.png)
+
+## en-et
+
+| Translator/Dataset | flores-dev | flores-test | wmt18 |
+| --- | --- | --- | --- |
+| bergamot | 25.60 | 25.70 | 25.20 |
+| google | 30.20 (+4.60, +17.97%) | 29.00 (+3.30, +12.84%) | 26.60 (+1.40, +5.56%) |
+| microsoft | 28.60 (+3.00, +11.72%) | 29.20 (+3.50, +13.62%) | 27.60 (+2.40, +9.52%) |
+
+![Results](img/en-et.png)
+
+## et-en
+
+| Translator/Dataset | flores-dev | flores-test | wmt18 |
+| --- | --- | --- | --- |
+| bergamot | 33.50 | 32.70 | 30.90 |
+| google | 38.30 (+4.80, +14.33%) | 37.00 (+4.30, +13.15%) | 32.10 (+1.20, +3.88%) |
+| microsoft | 37.40 (+3.90, +11.64%) | 37.00 (+4.30, +13.15%) | 34.10 (+3.20, +10.36%) |
+
+![Results](img/et-en.png)
 
 ## nb-en
 
@@ -95,82 +115,72 @@ Both absolute and relative differences in BLEU scores between Bergamot and other
 
 ![Results](img/bg-en.png)
 
-## et-en
-
-| Translator/Dataset | wmt18 | flores-dev | flores-test |
-| --- | --- | --- | --- |
-| bergamot | 30.90 | 33.50 | 32.70 |
-| google | 32.10 (+1.20, +3.88%) | 38.30 (+4.80, +14.33%) | 37.00 (+4.30, +13.15%) |
-| microsoft | 34.10 (+3.20, +10.36%) | 37.40 (+3.90, +11.64%) | 37.00 (+4.30, +13.15%) |
-
-![Results](img/et-en.png)
-
-## en-cs
-
-| Translator/Dataset | wmt10 | flores-dev | wmt11 | wmt18 | flores-test | wmt13 | wmt15 | wmt08 | wmt09 | wmt19 | wmt14 | wmt17 | wmt20 | wmt16 | wmt12 |
-| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| bergamot | 20.90 | 30.10 | 20.70 | 22.70 | 30.20 | 23.40 | 25.40 | 19.00 | 20.80 | 27.10 | 28.70 | 23.40 | 32.70 | 25.80 | 18.80 |
-| google | 22.40 (+1.50, +7.18%) | 34.10 (+4.00, +13.29%) | 23.00 (+2.30, +11.11%) | 24.40 (+1.70, +7.49%) | 34.40 (+4.20, +13.91%) | 25.20 (+1.80, +7.69%) | 26.80 (+1.40, +5.51%) | 20.50 (+1.50, +7.89%) | 22.60 (+1.80, +8.65%) | 27.20 (+0.10, +0.37%) | 31.20 (+2.50, +8.71%) | 24.70 (+1.30, +5.56%) | 35.50 (+2.80, +8.56%) | 28.30 (+2.50, +9.69%) | 20.70 (+1.90, +10.11%) |
-| microsoft | 24.30 (+3.40, +16.27%) | 33.50 (+3.40, +11.30%) | 25.30 (+4.60, +22.22%) | 24.90 (+2.20, +9.69%) | 34.00 (+3.80, +12.58%) | 27.70 (+4.30, +18.38%) | 27.40 (+2.00, +7.87%) | 22.60 (+3.60, +18.95%) | 25.00 (+4.20, +20.19%) | 27.20 (+0.10, +0.37%) | 31.90 (+3.20, +11.15%) | 25.60 (+2.20, +9.40%) | 34.10 (+1.40, +4.28%) | 29.90 (+4.10, +15.89%) | 22.90 (+4.10, +21.81%) |
-
-![Results](img/en-cs.png)
-
-## cs-en
-
-| Translator/Dataset | wmt17 | wmt10 | wmt18 | wmt09 | wmt14 | wmt11 | wmt16 | wmt13 | wmt20 | wmt08 | flores-dev | flores-test | wmt12 | wmt15 |
-| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| bergamot | 30.20 | 28.20 | 31.30 | 27.60 | 35.00 | 28.10 | 33.40 | 30.30 | 27.00 | 24.50 | 35.30 | 35.30 | 26.50 | 32.00 |
-| google | 31.20 (+1.00, +3.31%) | 30.50 (+2.30, +8.16%) | 32.10 (+0.80, +2.56%) | 29.90 (+2.30, +8.33%) | 38.00 (+3.00, +8.57%) | 30.20 (+2.10, +7.47%) | 34.80 (+1.40, +4.19%) | 32.40 (+2.10, +6.93%) | 28.40 (+1.40, +5.19%) | 26.30 (+1.80, +7.35%) | 38.60 (+3.30, +9.35%) | 39.00 (+3.70, +10.48%) | 28.60 (+2.10, +7.92%) | 33.60 (+1.60, +5.00%) |
-| microsoft | 33.60 (+3.40, +11.26%) | 30.70 (+2.50, +8.87%) | 34.30 (+3.00, +9.58%) | 29.60 (+2.00, +7.25%) | 39.90 (+4.90, +14.00%) | 30.90 (+2.80, +9.96%) | 38.30 (+4.90, +14.67%) | 33.40 (+3.10, +10.23%) | 27.60 (+0.60, +2.22%) | 26.40 (+1.90, +7.76%) | 40.00 (+4.70, +13.31%) | 40.30 (+5.00, +14.16%) | 29.70 (+3.20, +12.08%) | 34.70 (+2.70, +8.44%) |
-
-![Results](img/cs-en.png)
-
-## de-en
-
-| Translator/Dataset | wmt17 | wmt10 | wmt18 | wmt09 | wmt14 | wmt11 | wmt16 | wmt13 | wmt19 | wmt20 | wmt08 | flores-dev | flores-test | wmt12 | wmt15 | iwslt17 |
-| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| bergamot | 35.00 | 29.00 | 43.30 | 26.40 | 33.50 | 26.30 | 39.60 | 30.80 | 39.00 | 38.80 | 26.00 | 39.60 | 39.10 | 27.60 | 33.50 | 28.60 |
-| google | 38.70 (+3.70, +10.57%) | 32.10 (+3.10, +10.69%) | 46.20 (+2.90, +6.70%) | 27.20 (+0.80, +3.03%) | 37.40 (+3.90, +11.64%) | 27.30 (+1.00, +3.80%) | 42.30 (+2.70, +6.82%) | 32.40 (+1.60, +5.19%) | 41.10 (+2.10, +5.38%) | 41.80 (+3.00, +7.73%) | 27.60 (+1.60, +6.15%) | 43.10 (+3.50, +8.84%) | 42.80 (+3.70, +9.46%) | 29.50 (+1.90, +6.88%) | 36.10 (+2.60, +7.76%) | 30.10 (+1.50, +5.24%) |
-| microsoft | 40.80 (+5.80, +16.57%) | 33.40 (+4.40, +15.17%) | 49.60 (+6.30, +14.55%) | 29.10 (+2.70, +10.23%) | 39.20 (+5.70, +17.01%) | 29.20 (+2.90, +11.03%) | 46.30 (+6.70, +16.92%) | 34.30 (+3.50, +11.36%) | 43.80 (+4.80, +12.31%) | 43.60 (+4.80, +12.37%) | 29.40 (+3.40, +13.08%) | 44.90 (+5.30, +13.38%) | 45.80 (+6.70, +17.14%) | 31.30 (+3.70, +13.41%) | 38.10 (+4.60, +13.73%) | 32.50 (+3.90, +13.64%) |
-
-![Results](img/de-en.png)
-
 ## en-es
 
-| Translator/Dataset | wmt13 | flores-test | wmt12 | wmt08 | wmt11 | flores-dev | wmt10 | wmt09 |
+| Translator/Dataset | wmt08 | wmt11 | wmt09 | flores-dev | wmt10 | wmt13 | wmt12 | flores-test |
 | --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| bergamot | 34.80 | 26.20 | 38.90 | 29.00 | 37.90 | 25.90 | 36.70 | 29.90 |
-| google | 36.90 (+2.10, +6.03%) | 30.40 (+4.20, +16.03%) | 40.50 (+1.60, +4.11%) | 30.00 (+1.00, +3.45%) | 39.90 (+2.00, +5.28%) | 30.50 (+4.60, +17.76%) | 38.80 (+2.10, +5.72%) | 30.90 (+1.00, +3.34%) |
-| microsoft | 35.70 (+0.90, +2.59%) | 28.50 (+2.30, +8.78%) | 40.00 (+1.10, +2.83%) | 29.90 (+0.90, +3.10%) | 39.10 (+1.20, +3.17%) | 28.40 (+2.50, +9.65%) | 37.80 (+1.10, +3.00%) | 30.70 (+0.80, +2.68%) |
+| bergamot | 29.00 | 37.90 | 29.90 | 25.90 | 36.70 | 34.80 | 38.90 | 26.20 |
+| google | 30.00 (+1.00, +3.45%) | 39.90 (+2.00, +5.28%) | 30.90 (+1.00, +3.34%) | 30.50 (+4.60, +17.76%) | 38.80 (+2.10, +5.72%) | 36.90 (+2.10, +6.03%) | 40.50 (+1.60, +4.11%) | 30.40 (+4.20, +16.03%) |
+| microsoft | 29.90 (+0.90, +3.10%) | 39.10 (+1.20, +3.17%) | 30.70 (+0.80, +2.68%) | 28.40 (+2.50, +9.65%) | 37.80 (+1.10, +3.00%) | 35.70 (+0.90, +2.59%) | 40.00 (+1.10, +2.83%) | 28.50 (+2.30, +8.78%) |
 
 ![Results](img/en-es.png)
 
-## en-et
-
-| Translator/Dataset | wmt18 | flores-test | flores-dev |
-| --- | --- | --- | --- |
-| bergamot | 25.20 | 25.70 | 25.60 |
-| google | 26.60 (+1.40, +5.56%) | 29.00 (+3.30, +12.84%) | 30.20 (+4.60, +17.97%) |
-| microsoft | 27.60 (+2.40, +9.52%) | 29.20 (+3.50, +13.62%) | 28.60 (+3.00, +11.72%) |
-
-![Results](img/en-et.png)
-
 ## en-bg
 
-| Translator/Dataset | flores-test | flores-dev |
+| Translator/Dataset | flores-dev | flores-test |
 | --- | --- | --- |
-| bergamot | 42.20 | 42.00 |
-| google | 45.10 (+2.90, +6.87%) | 44.10 (+2.10, +5.00%) |
-| microsoft | 39.10 (-3.10, -7.35%) | 38.00 (-4.00, -9.52%) |
+| bergamot | 42.00 | 42.20 |
+| google | 44.10 (+2.10, +5.00%) | 45.10 (+2.90, +6.87%) |
+| microsoft | 38.00 (-4.00, -9.52%) | 39.10 (-3.10, -7.35%) |
 
 ![Results](img/en-bg.png)
 
+## en-cs
+
+| Translator/Dataset | wmt12 | flores-dev | wmt19 | wmt14 | wmt17 | wmt20 | wmt15 | wmt08 | wmt11 | wmt18 | wmt09 | wmt10 | flores-test | wmt13 | wmt16 |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| bergamot | 18.80 | 30.10 | 27.10 | 28.70 | 23.40 | 32.70 | 25.40 | 19.00 | 20.70 | 22.70 | 20.80 | 20.90 | 30.20 | 23.40 | 25.80 |
+| google | 20.70 (+1.90, +10.11%) | 34.10 (+4.00, +13.29%) | 27.20 (+0.10, +0.37%) | 31.20 (+2.50, +8.71%) | 24.70 (+1.30, +5.56%) | 35.50 (+2.80, +8.56%) | 26.80 (+1.40, +5.51%) | 20.50 (+1.50, +7.89%) | 23.00 (+2.30, +11.11%) | 24.40 (+1.70, +7.49%) | 22.60 (+1.80, +8.65%) | 22.40 (+1.50, +7.18%) | 34.40 (+4.20, +13.91%) | 25.20 (+1.80, +7.69%) | 28.30 (+2.50, +9.69%) |
+| microsoft | 22.90 (+4.10, +21.81%) | 33.50 (+3.40, +11.30%) | 27.20 (+0.10, +0.37%) | 31.90 (+3.20, +11.15%) | 25.60 (+2.20, +9.40%) | 34.10 (+1.40, +4.28%) | 27.40 (+2.00, +7.87%) | 22.60 (+3.60, +18.95%) | 25.30 (+4.60, +22.22%) | 24.90 (+2.20, +9.69%) | 25.00 (+4.20, +20.19%) | 24.30 (+3.40, +16.27%) | 34.00 (+3.80, +12.58%) | 27.70 (+4.30, +18.38%) | 29.90 (+4.10, +15.89%) |
+
+![Results](img/en-cs.png)
+
+## de-en
+
+| Translator/Dataset | wmt08 | wmt19 | wmt17 | iwslt17 | wmt10 | flores-dev | flores-test | wmt12 | wmt11 | wmt14 | wmt15 | wmt16 | wmt13 | wmt18 | wmt09 | wmt20 |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| bergamot | 26.00 | 39.00 | 35.00 | 28.60 | 29.00 | 39.60 | 39.10 | 27.60 | 26.30 | 33.50 | 33.50 | 39.60 | 30.80 | 43.30 | 26.40 | 38.80 |
+| google | 27.60 (+1.60, +6.15%) | 41.10 (+2.10, +5.38%) | 38.70 (+3.70, +10.57%) | 30.10 (+1.50, +5.24%) | 32.10 (+3.10, +10.69%) | 43.10 (+3.50, +8.84%) | 42.80 (+3.70, +9.46%) | 29.50 (+1.90, +6.88%) | 27.30 (+1.00, +3.80%) | 37.40 (+3.90, +11.64%) | 36.10 (+2.60, +7.76%) | 42.30 (+2.70, +6.82%) | 32.40 (+1.60, +5.19%) | 46.20 (+2.90, +6.70%) | 27.20 (+0.80, +3.03%) | 41.80 (+3.00, +7.73%) |
+| microsoft | 29.40 (+3.40, +13.08%) | 43.80 (+4.80, +12.31%) | 40.80 (+5.80, +16.57%) | 32.50 (+3.90, +13.64%) | 33.40 (+4.40, +15.17%) | 44.90 (+5.30, +13.38%) | 45.80 (+6.70, +17.14%) | 31.30 (+3.70, +13.41%) | 29.20 (+2.90, +11.03%) | 39.20 (+5.70, +17.01%) | 38.10 (+4.60, +13.73%) | 46.30 (+6.70, +16.92%) | 34.30 (+3.50, +11.36%) | 49.60 (+6.30, +14.55%) | 29.10 (+2.70, +10.23%) | 43.60 (+4.80, +12.37%) |
+
+![Results](img/de-en.png)
+
+## pt-en
+
+| Translator/Dataset | mtedx_test | flores-dev | flores-test |
+| --- | --- | --- | --- |
+| bergamot | 40.20 | 47.80 | 46.60 |
+| google | 39.10 (-1.10, -2.74%) | 50.40 (+2.60, +5.44%) | 50.30 (+3.70, +7.94%) |
+| microsoft | 41.00 (+0.80, +1.99%) | 49.80 (+2.00, +4.18%) | 48.60 (+2.00, +4.29%) |
+
+![Results](img/pt-en.png)
+
+## es-en
+
+| Translator/Dataset | wmt08 | mtedx_test | wmt10 | flores-dev | flores-test | wmt12 | wmt11 | wmt13 | wmt09 |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| bergamot | 27.30 | 36.80 | 35.80 | 27.50 | 26.80 | 38.30 | 34.30 | 35.20 | 29.40 |
+| google | 28.30 (+1.00, +3.66%) | 35.40 (-1.40, -3.80%) | 37.00 (+1.20, +3.35%) | 30.50 (+3.00, +10.91%) | 30.30 (+3.50, +13.06%) | 38.80 (+0.50, +1.31%) | 35.20 (+0.90, +2.62%) | 35.70 (+0.50, +1.42%) | 31.60 (+2.20, +7.48%) |
+| microsoft | 26.80 (-0.50, -1.83%) | 37.60 (+0.80, +2.17%) | 35.40 (-0.40, -1.12%) | 30.30 (+2.80, +10.18%) | 29.90 (+3.10, +11.57%) | 37.80 (-0.50, -1.31%) | 33.70 (-0.60, -1.75%) | 35.30 (+0.10, +0.28%) | 29.60 (+0.20, +0.68%) |
+
+![Results](img/es-en.png)
+
 ## en-de
 
-| Translator/Dataset | wmt12 | wmt16 | wmt11 | wmt15 | wmt13 | iwslt17 | wmt08 | wmt10 | wmt19 | wmt14 | wmt20 | flores-test | wmt18 | wmt17 | wmt09 | flores-dev |
+| Translator/Dataset | flores-test | iwslt17 | wmt20 | wmt08 | wmt12 | wmt19 | wmt10 | wmt11 | flores-dev | wmt09 | wmt13 | wmt17 | wmt16 | wmt18 | wmt14 | wmt15 |
 | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| bergamot | 24.30 | 40.00 | 23.40 | 33.10 | 28.20 | 26.70 | 23.60 | 26.80 | 44.50 | 29.80 | 35.70 | 38.80 | 47.70 | 32.00 | 23.00 | 38.80 |
-| google | 24.70 (+0.40, +1.65%) | 38.60 (-1.40, -3.50%) | 24.10 (+0.70, +2.99%) | 33.70 (+0.60, +1.81%) | 28.80 (+0.60, +2.13%) | 28.90 (+2.20, +8.24%) | 23.70 (+0.10, +0.42%) | 26.50 (-0.30, -1.12%) | 43.50 (-1.00, -2.25%) | 30.90 (+1.10, +3.69%) | 36.50 (+0.80, +2.24%) | 42.30 (+3.50, +9.02%) | 47.80 (+0.10, +0.21%) | 31.50 (-0.50, -1.56%) | 23.60 (+0.60, +2.61%) | 43.70 (+4.90, +12.63%) |
-| microsoft | 25.30 (+1.00, +4.12%) | 40.50 (+0.50, +1.25%) | 23.70 (+0.30, +1.28%) | 34.30 (+1.20, +3.63%) | 28.80 (+0.60, +2.13%) | 28.20 (+1.50, +5.62%) | 24.00 (+0.40, +1.69%) | 27.20 (+0.40, +1.49%) | 43.80 (-0.70, -1.57%) | 32.20 (+2.40, +8.05%) | 36.10 (+0.40, +1.12%) | 42.90 (+4.10, +10.57%) | 48.70 (+1.00, +2.10%) | 33.10 (+1.10, +3.44%) | 23.90 (+0.90, +3.91%) | 44.00 (+5.20, +13.40%) |
+| bergamot | 38.80 | 26.70 | 35.70 | 23.60 | 24.30 | 44.50 | 26.80 | 23.40 | 38.80 | 23.00 | 28.20 | 32.00 | 40.00 | 47.70 | 29.80 | 33.10 |
+| google | 42.30 (+3.50, +9.02%) | 28.90 (+2.20, +8.24%) | 36.50 (+0.80, +2.24%) | 23.70 (+0.10, +0.42%) | 24.70 (+0.40, +1.65%) | 43.50 (-1.00, -2.25%) | 26.50 (-0.30, -1.12%) | 24.10 (+0.70, +2.99%) | 43.70 (+4.90, +12.63%) | 23.60 (+0.60, +2.61%) | 28.80 (+0.60, +2.13%) | 31.50 (-0.50, -1.56%) | 38.60 (-1.40, -3.50%) | 47.80 (+0.10, +0.21%) | 30.90 (+1.10, +3.69%) | 33.70 (+0.60, +1.81%) |
+| microsoft | 42.90 (+4.10, +10.57%) | 28.20 (+1.50, +5.62%) | 36.10 (+0.40, +1.12%) | 24.00 (+0.40, +1.69%) | 25.30 (+1.00, +4.12%) | 43.80 (-0.70, -1.57%) | 27.20 (+0.40, +1.49%) | 23.70 (+0.30, +1.28%) | 44.00 (+5.20, +13.40%) | 23.90 (+0.90, +3.91%) | 28.80 (+0.60, +2.13%) | 33.10 (+1.10, +3.44%) | 40.50 (+0.50, +1.25%) | 48.70 (+1.00, +2.10%) | 32.20 (+2.40, +8.05%) | 34.30 (+1.20, +3.63%) |
 
 ![Results](img/en-de.png)

models/dev/pten/lex.50.50.pten.s2t.bin.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:74ae550c09dc96c09789ae66d7a5047a88fa666a48d86a25da8374a1e139dc7b
-size 2733800

models/dev/pten/model.pten.intgemm.alphas.bin.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:19a8878f2faf8cfd55e76017e2bc6979bdb834c2a751ad12f3f96c64cd9c95ec
-size 13584764

models/dev/pten/vocab.pten.spm.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3c9d9c5fab15c33fc94146bcf34937b2248e744217d2e9b67d80ed01d86df57d
-size 406730

models/prod/pten/lex.50.50.pten.s2t.bin.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:591c5f0e825bd45b7878320d817c19e07052c8f07447fbed86e0adcd590992f9
+size 2639080

models/prod/pten/model.pten.intgemm.alphas.bin.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a2154a22b49d74fa62ce09a5dbcf0e5bc2653a83c882334b2e7392e8eb6a402
+size 13204180

models/prod/pten/vocab.pten.spm.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41d51de148610731a8508273636f22430e783a955a2c4d0210ef482ae22764a4
+size 410584

registry.json

@@ -273,6 +273,29 @@
       "modelType": "prod"
     }
   },
+  "pten": {
+    "model": {
+      "name": "model.pten.intgemm.alphas.bin",
+      "size": 17140899,
+      "estimatedCompressedSize": 13204180,
+      "expectedSha256Hash": "b4a1fd101f59ca258bfb051a3a4e5762497b9f91d7e74f28b7a2d68dc075f8fa",
+      "modelType": "prod"
+    },
+    "lex": {
+      "name": "lex.50.50.pten.s2t.bin",
+      "size": 4801740,
+      "estimatedCompressedSize": 2639080,
+      "expectedSha256Hash": "702d9b09fbc020168998bb81afc927e74262f5a1455e9b443608a5d8efa4b52d",
+      "modelType": "prod"
+    },
+    "vocab": {
+      "name": "vocab.pten.spm",
+      "size": 817234,
+      "estimatedCompressedSize": 410584,
+      "expectedSha256Hash": "2326a27577b5ffa0b822c55fe850986961728097521de35fcc186f7d6dce72d4",
+      "modelType": "prod"
+    }
+  },
   "enfa": {
     "model": {
       "name": "model.enfa.intgemm.alphas.bin",
@@ -457,29 +480,6 @@
       "modelType": "dev"
     }
   },
-  "pten": {
-    "model": {
-      "name": "model.pten.intgemm.alphas.bin",
-      "size": 17140836,
-      "estimatedCompressedSize": 13584764,
-      "expectedSha256Hash": "6c3b7af01772022a19712410c63342ba581468c2f1aac34d7488409c4043e697",
-      "modelType": "dev"
-    },
-    "lex": {
-      "name": "lex.50.50.pten.s2t.bin",
-      "size": 5001420,
-      "estimatedCompressedSize": 2733800,
-      "expectedSha256Hash": "212ed0ae44a6f920cd6d17ca02f0a523ba6c4b0ef5078ae310c20bc4c51484c5",
-      "modelType": "dev"
-    },
-    "vocab": {
-      "name": "vocab.pten.spm",
-      "size": 812889,
-      "estimatedCompressedSize": 406730,
-      "expectedSha256Hash": "8389979e3c965688b07aeb712a7e44406e5dcdb2b84087229d26fcc71448c4ed",
-      "modelType": "dev"
-    }
-  },
   "ruen": {
     "model": {
       "name": "model.ruen.intgemm.alphas.bin",