From 336b7905a8205ee3fcb2f7f04c3e1f1e03f7bdfa Mon Sep 17 00:00:00 2001 From: LoafyLemon Date: Mon, 8 Jan 2024 14:44:43 +0000 Subject: [PATCH] Translation tools --- game/scripts/utility/translation.rpy | 305 +++++++++++++-------------- 1 file changed, 143 insertions(+), 162 deletions(-) diff --git a/game/scripts/utility/translation.rpy b/game/scripts/utility/translation.rpy index e4f68e41..9492eff3 100644 --- a/game/scripts/utility/translation.rpy +++ b/game/scripts/utility/translation.rpy @@ -3,208 +3,189 @@ rpy python 3 init python in cli: import os import collections - import itertools - from renpy.translation import quote_unicode - from renpy.parser import elide_filename - tl_file_cache = {} + def is_random_block(t): + return any("random:" in n.get_code() or "block:" in n.get_code() for n in t.block) - def open_tl_file(fn, mode): + def write_translates(filename, language, filter): + generation = renpy.translation.generation + fn, common = generation.shorten_filename(filename) - if fn in tl_file_cache: - f = tl_file_cache[fn] + # The common directory should not have dialogue in it. + if common: + return - if f.mode == mode: - return f + tl_filename = os.path.join(renpy.config.gamedir, renpy.config.tl_directory, language, fn) # type: ignore - # elif not f.closed: - # # PY 2 only - # f.close() + if tl_filename[-1] == "m": + tl_filename = tl_filename[:-1] - if not os.path.exists(fn): - dn = os.path.dirname(fn) + if language == "None": + language = None - try: - os.makedirs(dn) - except Exception: - pass + translator = renpy.game.script.translator - f = open(fn, mode, encoding="utf-8") - f.write(u"\ufeff") + # Truncate the file before rewrite + if os.path.exists(tl_filename): + with open(tl_filename, "w") as f: + f.truncate(0) + # Re-add everything with updated pointers and lines + for label, t in translator.file_translates[filename]: + + # if (t.identifier, language) in translator.language_translates: + # continue + + if hasattr(t, "alternate"): + if (t.alternate, language) in translator.language_translates: + continue + + if generation.is_empty_extend(t): + continue + + # if is_random_block(t): + # continue + + if hasattr(t, "translatable") and t.translatable == False: + block = t.block + print(f"Found untranslatable node block: {block}") + continue + + label = label or "" + + with open(tl_filename, "a") as f: + f.write("# {}:{}\n".format(t.filename, t.linenumber)) + f.write("translate {} {}:\n".format(language, t.identifier.replace('.', '_'))) + f.write("\n") + + # Grab original code block + # for n in t.block: + # f.write(" # " + n.get_code() + "\n") + + translated = translator.language_translates.get((t.identifier, language)) or t + + # Add commented code and an optional TODO if there's a mismatch. + for block1, block2 in zip(t.block, translated.block): + old = block1.get_code() + new = block2.get_code() + + suffix = "\n" if old == new else " # TODO: Updated dialogue.\n" # TODO: This does not function, probably due to translated var being overridden. + f.write(" # " + old + suffix) + + # Re-apply translation + for n in translated.block: + f.write(" " + n.get_code(filter) + "\n") + + f.write("\n") + + def append_unsanctioned_strings(language, seen): + """Used to extract pre-existing strings, which can be a part of an unsanctioned translation. We want to keep those.""" + + generation = renpy.translation.generation + stl = renpy.game.script.translator.strings[language] + + for old, new in stl.translations.items(): + + if not new or new == old: + continue + + if old in seen: # TODO: Unsanctioned translations sometimes double for some reason despite this check + continue + + if old not in stl.translation_loc: + continue + + filename, linenumber = stl.translation_loc[old] + tlfn = os.path.join(renpy.config.basedir, filename) + + with open(tlfn, "a+") as f: + f.seek(0) + contents = f.read() + prefix = u"translate {} strings:\n".format(language) + + if prefix not in contents: + f.write(prefix) + f.write(u"\n") + + # f.write(u" # {}:{}\n".format(generation.elide_filename(filename), linenumber)) # TODO: This gives incorrect results; Seek alternative methods. + f.write(u" # Unsanctioned translation\n") + f.write(u" old \"{}\"\n".format(generation.quote_unicode(old))) + f.write(u" new \"{}\"\n".format(generation.quote_unicode(new))) + f.write(u"\n") + + def write_strings(language, filter, min_priority, max_priority, common_only, only_strings=[]): # @ReservedAssignment + generation = renpy.translation.generation + + if language == "None": + stl = renpy.game.script.translator.strings[None] # @UndefinedVariable else: - f = open(fn, mode, encoding="utf-8") + stl = renpy.game.script.translator.strings[language] # @UndefinedVariable - tl_file_cache[fn] = f - - return f - - def scan_strings(strings, min_priority=0, max_priority=299, common_only=False): - - strings.sort(key=lambda s : s.sort_key) - - rv = [ ] - seen = set() + strings = renpy.translation.scanstrings.scan(min_priority, max_priority, common_only) + stringfiles = collections.defaultdict(list) for s in strings: - if s.priority < min_priority: - continue - - if s.priority > max_priority: - continue - - if common_only and not s.common: - continue - - if s.text in seen: - continue - - seen.add(s.text) - rv.append(s) - - return rv - - def write_strings(language, strings): - - stl = renpy.game.script.translator.strings[language] - stringfiles = collections.defaultdict(list) - seen = set() - - nstrings = len(strings)-1 - - for i, s in enumerate(strings): - n = round(float(i)/(nstrings)*100) - print("\rGenerating strings for {} ... Total progress:{} % ... Stage 2/2".format(language, n), end="") - - tlfn = renpy.translation.generation.translation_filename(s) + tlfn = generation.translation_filename(s) if tlfn is None: continue - if s.text in seen: + # Already seen. + if s.text in stl.translations: continue - seen.add(s.text) - if language == "None" and tlfn == "common.rpy": tlfn = "common.rpym" + if only_strings and s.text not in only_strings: + continue + stringfiles[tlfn].append(s) - for tlfn, sl in list(stringfiles.items()): + seen = set() + + for tlfn, sl in stringfiles.items(): + + # sl.sort(key=lambda s : (s.filename, s.line)) + tlfn = os.path.join(renpy.config.gamedir, renpy.config.tl_directory, language, tlfn) - f = open_tl_file(tlfn, mode="w") - f.write(u"translate {} strings:\n".format(language)) - f.write(u"\n") + with open(tlfn, "a") as f: - for s in sl: - original = s.text - translation = stl.translate(s.text) # Keeps translated strings - - f.write(u" # {}:{}\n".format(elide_filename(s.filename), s.line)) - f.write(u" old \"{}\"\n".format(quote_unicode(original))) - f.write(u" new \"{}\"\n".format(quote_unicode(translation))) + f.write(u"translate {} strings:\n".format(language)) f.write(u"\n") + for s in sl: + seen.add(s.text) + text = filter(s.text) + + f.write(u" # {}:{}\n".format(generation.elide_filename(s.filename), s.line)) + f.write(u" old \"{}\"\n".format(generation.quote_unicode(s.text))) + f.write(u" new \"{}\"\n".format(generation.quote_unicode(text))) + f.write(u"\n") + + # Re-add unsanctioned translation strings + append_unsanctioned_strings(language, seen) + def retranslate(): translator = renpy.game.script.translator generation = renpy.translation.generation - scanstrings = renpy.translation.scanstrings - - parser = renpy.arguments.ArgumentParser() - parser.add_argument("--rebuild", action="store_true", help="Rebuilds the translation pointers and adds missing entries.") - parser.add_argument("--empty", action="store_true", help="When combined with rebuild parameter, it will insert an empty string into added entires. Does nothing on its own.") - parser.add_argument("--clean", action="store_true", help="Removes translation files that are no longer present within the game files.") - parser.add_argument("--dry", action="store_true", help="Simulates the removal of translation files that are no longer present within the game files.") - parser.add_argument("--include-mods", action="store_true", help="Include mod files when generating translations.") - args = parser.parse_args() + filter = generation.null_filter scripts = generation.translate_list_files() - - if not args.include_mods: - mods_dir = os.path.join(renpy.config.gamedir, "mods") - scripts = [f for f in scripts if not f.startswith(mods_dir)] - - strings = [] nscripts = len(scripts)-1 + nlanguages = len(translator.languages) - for i, filepath in enumerate(scripts): - n = round(float(i)/(nscripts)*100) + for i, language in enumerate(translator.languages): - for language in translator.languages: + for j, filename in enumerate(scripts): + n = round(float(j)/nscripts*100) + print("\rGenerating translation for {} {}% ({}/{})...".format(language, n, i+1, nlanguages), end="") - print("\rGenerating dialogues for {} ... Total progress:{} % ... Stage 1/2".format(language, n), end="") + write_translates(filename, language, filter) - for _, trans in translator.file_translates[filepath]: - - trans_new = translator.language_translates.get((trans.identifier, language)) - filter = generation.null_filter - - if trans_new is None: - if not args.rebuild: - continue - - trans_new = trans - - if args.empty: - filter = generation.empty_filter - - if hasattr(trans, "alternate") and (trans.alternate, language) in translator.language_translates: - continue - - fp = os.path.relpath(filepath, renpy.config.gamedir) - fp = os.path.join(renpy.config.gamedir, renpy.config.tl_directory, language, fp) - - f = open_tl_file(fp, mode="w") - - f.write(u"# {}:{}\n".format(trans.filename, trans.linenumber)) - f.write(u"translate {} {}:\n".format(language, trans.identifier.replace(".", "_"))) - f.write(u"\n") - - for n in trans.block: - f.write(u" # " + n.get_code() + "\n") - - for n in trans_new.block: - f.write(u" " + n.get_code(filter) + "\n") - - f.write(u"\n") - - strings.extend(scanstrings.scan_strings(filepath)) - - strings = scan_strings(strings) - - for language in translator.languages: - - write_strings(language, strings) - - if args.clean: - translations = [f for f in renpy.list_files() if f.startswith(renpy.config.tl_directory) and f.endswith((".rpy", ".rpym"))] - - for filepath in translations: - fn = os.path.basename(filepath) - fp = os.path.relpath(filepath, os.path.join(renpy.config.tl_directory, language)) - fp = os.path.join(renpy.config.gamedir, fp) - - if fp not in scripts: - if fn == "common.rpy": - continue - - if filepath in tl_file_cache: - f = tl_file_cache.pop(tl_file_cache) - f.close() - - if args.dry: - print("Removal required: {}".format(filepath)) - else: - os.unlink(filepath) - os.unlink(filepath + "c") - - for f in list(tl_file_cache.values()): - f.close() - - tl_file_cache.clear() + write_strings(language, filter, 0, 299, False) return False