From bf94aa93cf2a81f010456889963e2bd680ff688a Mon Sep 17 00:00:00 2001
From: tophf <tophf@gmx.com>
Date: Wed, 12 Jul 2017 11:27:57 +0300
Subject: [PATCH] pull_locales_postprocess: deduplicate and placeholderize

---
 pull_locales.sh                               |  2 +-
 ...eholders.py => pull_locales_postprocess.py | 23 +++++++++++++++----
 2 files changed, 20 insertions(+), 5 deletions(-)
 rename fill_locale_placeholders.py => pull_locales_postprocess.py (57%)

diff --git a/pull_locales.sh b/pull_locales.sh
index a1e4531d..a53ccee3 100755
--- a/pull_locales.sh
+++ b/pull_locales.sh
@@ -1,2 +1,2 @@
 ruby pull_locales.rb
-python fill_locale_placeholders.py
+python pull_locales_postprocess.py
diff --git a/fill_locale_placeholders.py b/pull_locales_postprocess.py
similarity index 57%
rename from fill_locale_placeholders.py
rename to pull_locales_postprocess.py
index 37027738..a57ebc2d 100644
--- a/fill_locale_placeholders.py
+++ b/pull_locales_postprocess.py
@@ -3,23 +3,34 @@ import io, os, json, re
 from collections import OrderedDict
 
 with io.open('_locales/en/messages.json', 'r', encoding='utf-8') as f:
-    english_placeholders = [(k, v['placeholders']) for k,v in json.load(f).items()
+    items = json.load(f).items()
+    english = [(k, v['message']) for k, v in items if 'message' in v]
+    english_placeholders = [(k, v['placeholders']) for k,v in items
                             if 'placeholders' in v]
 
 for locale_name in os.listdir('_locales'):
     if locale_name == 'en':
         continue
+    if not re.match(r'^\w{2}(_\w{2,3})?$', locale_name):
+        print('Skipped %s: not a locale dir' % locale_name)
+        continue
     loc_path = '_locales/' + locale_name + '/messages.json'
     with io.open(loc_path, 'r+', encoding='utf-8') as f:
         loc = json.load(f, object_pairs_hook=OrderedDict)
 
+        deduplicated = 0
+        for msgId, message in english:
+            if msgId in loc and loc[msgId].get('message', '') == message:
+                del loc[msgId]
+                deduplicated += 1
+
         changed = 0
         for msgId, placeholder in english_placeholders:
-            if msgId in loc and cmp(placeholder, loc[msgId].get('placeholders', None))!=0:
+            if msgId in loc and cmp(placeholder, loc[msgId].get('placeholders', None)) != 0:
                 loc[msgId]['placeholders'] = placeholder
                 changed += 1
 
-        if changed > 0:
+        if deduplicated > 0 or changed > 0:
             f.seek(0)
             json_str = json.dumps(loc, indent=1, ensure_ascii=False,
                                   separators=(',', ': '), encoding='utf-8')
@@ -27,4 +38,8 @@ for locale_name in os.listdir('_locales'):
                                json_str, flags=re.MULTILINE)
             f.write(json_tabs)
             f.truncate()
-            print 'Placeholders added to %s: %d' % (locale_name, changed)
+            print('%s: %d deduplicated%s' % (
+                locale_name,
+                deduplicated,
+                ', %d placeholder(s) added' % changed if changed else ''
+            ))