Updated script for better splitting of synonyms

2024-10-30 10:51:05 -07:00
parent 485c5175b8
commit b21e80f708
17 changed files with 11401 additions and 27093 deletions
--- a/json/da.json
+++ b/json/da.json
--- a/json/de.json
+++ b/json/de.json
--- a/json/en.json
+++ b/json/en.json
--- a/json/es.json
+++ b/json/es.json
--- a/json/fi.json
+++ b/json/fi.json
--- a/json/fr.json
+++ b/json/fr.json
--- a/json/it.json
+++ b/json/it.json
--- a/json/nl.json
+++ b/json/nl.json
--- a/json/no.json
+++ b/json/no.json
--- a/json/pl.json
+++ b/json/pl.json
--- a/json/pt-br.json
+++ b/json/pt-br.json
--- a/json/pt.json
+++ b/json/pt.json
--- a/json/ru.json
+++ b/json/ru.json
--- a/json/sv.json
+++ b/json/sv.json
--- a/json/tr.json
+++ b/json/tr.json
--- a/json/ua.json
+++ b/json/ua.json
--- a/tools/ucsxls2json.py
+++ b/tools/ucsxls2json.py
@@ -80,7 +80,10 @@ for lang in langs:
                syns_raw = category['Synonyms']
                assert type(syns_raw) == str, \
                        f"Synonym list (lang: {lang}, {category['CatID']}) was not readable"
-                syn_list = re.split(r'\W+', syns_raw)
+                split_pattern = r',\s*'
+                if lang in ['zh','ar','kr','ja','tw']:
+                    split_pattern = r'\W+'
+                syn_list = re.split(split_pattern, syns_raw)
                category['Synonyms'] = [s.lower() for s in syn_list]
        
        schedule.append(category)