diff options
Diffstat (limited to 'dev-util/itstool/files/itstool-2.0.2-python3.patch')
| -rw-r--r-- | dev-util/itstool/files/itstool-2.0.2-python3.patch | 622 |
1 files changed, 622 insertions, 0 deletions
diff --git a/dev-util/itstool/files/itstool-2.0.2-python3.patch b/dev-util/itstool/files/itstool-2.0.2-python3.patch new file mode 100644 index 000000000..9c2ed3237 --- /dev/null +++ b/dev-util/itstool/files/itstool-2.0.2-python3.patch @@ -0,0 +1,622 @@ +From c2e1824671747030234dab895400699cbc1d99be Mon Sep 17 00:00:00 2001 +From: Virgil Dupras <hsoft@hardcoded.net> +Date: Sun, 15 Mar 2015 20:04:37 -0400 +Subject: [PATCH 1/2] Add python 3 support + +--- + itstool.in | 180 +++++++++++++++++++++++++++++++++++-------------------------- + 1 file changed, 103 insertions(+), 77 deletions(-) + +diff --git a/itstool.in b/itstool.in +index d0e76a1..1a59a5a 100755 +--- a/itstool.in ++++ b/itstool.in +@@ -16,6 +16,8 @@ + # with ITS Tool; if not, write to the Free Software Foundation, 59 Temple + # Place, Suite 330, Boston, MA 0211-1307 USA. + # ++from __future__ import print_function ++from __future__ import unicode_literals + + VERSION="@VERSION@" + DATADIR="@DATADIR@" +@@ -29,6 +31,22 @@ import os.path + import re + import sys + import time ++import io ++ ++PY3 = sys.version_info[0] == 3 ++if PY3: ++ string_types = str, ++ def ustr(s, encoding=None): ++ if isinstance(s, str): ++ return s ++ elif encoding: ++ return str(s, encoding) ++ else: ++ return str(s) ++ ustr_type = str ++else: ++ string_types = basestring, ++ ustr = ustr_type = unicode + + NS_ITS = 'http://www.w3.org/2005/11/its' + NS_ITST = 'http://itstool.org/extensions/' +@@ -81,14 +99,14 @@ class MessageList (object): + return self._by_node.get(node, None) + + def get_nodes_with_messages (self): +- return self._by_node.keys() ++ return list(self._by_node.keys()) + + def output (self, out): + msgs = [] + msgdict = {} + for msg in self._messages: + key = (msg.get_context(), msg.get_string()) +- if msgdict.has_key(key): ++ if key in msgdict: + for source in msg.get_sources(): + msgdict[key].add_source(source) + for marker in msg.get_markers(): +@@ -105,7 +123,7 @@ class MessageList (object): + msgdict[key].set_locale_filter('%s, %s' % (locale, msg.get_locale_filter())) + else: + msgdict[key].set_locale_filter(msg.get_locale_filter()) +- ++ + else: + msgs.append(msg) + msgdict[key] = msg +@@ -121,13 +139,13 @@ class MessageList (object): + out.write('"Content-Transfer-Encoding: 8bit\\n"\n') + out.write('\n') + for msg in msgs: +- out.write(msg.format().encode('utf-8')) ++ out.write(msg.format()) + out.write('\n') + + + class Comment (object): + def __init__ (self, text): +- self._text = str(text) ++ self._text = ustr(text) + assert(text is not None) + self._markers = [] + +@@ -141,10 +159,10 @@ class Comment (object): + return self._text + + def format (self): +- ret = u'' ++ ret = '' + markers = {} + for marker in self._markers: +- if not markers.has_key(marker): ++ if marker not in markers: + ret += '#. (itstool) comment: ' + marker + '\n' + markers[marker] = marker + if '\n' in self._text: +@@ -154,7 +172,7 @@ class Comment (object): + doadd = True + if not doadd: + continue +- ret += u'#. %s\n' % line ++ ret += '#. %s\n' % line + else: + text = self._text + while len(text) > 72: +@@ -163,7 +181,7 @@ class Comment (object): + j = text.find(' ') + if j == -1: + break +- ret += u'#. %s\n' % text[:j] ++ ret += '#. %s\n' % text[:j] + text = text[j+1:] + ret += '#. %s\n' % text + return ret +@@ -190,16 +208,16 @@ class Message (object): + class Placeholder (object): + def __init__ (self, node): + self.node = node +- self.name = unicode(node.name, 'utf-8') ++ self.name = ustr(node.name, 'utf-8') + + def escape (self, text): + return text.replace('\\','\\\\').replace('"', "\\\"").replace("\n","\\n").replace("\t","\\t") + + def add_text (self, text): +- if len(self._message) == 0 or not(isinstance(self._message[-1], basestring)): ++ if len(self._message) == 0 or not(isinstance(self._message[-1], string_types)): + self._message.append('') +- if not isinstance(text, unicode): +- text = unicode(text, 'utf-8') ++ if not isinstance(text, ustr_type): ++ text = ustr(text, 'utf-8') + self._message[-1] += text.replace('&', '&').replace('<', '<').replace('>', '>') + if re.sub('\s+', ' ', text).strip() != '': + self._empty = False +@@ -216,25 +234,25 @@ class Message (object): + def get_placeholder (self, name): + placeholder = 1 + for holder in self._placeholders: +- holdername = u'%s-%i' % (holder.name, placeholder) +- if holdername == unicode(name, 'utf-8'): ++ holdername = '%s-%i' % (holder.name, placeholder) ++ if holdername == ustr(name, 'utf-8'): + return holder + placeholder += 1 + + def add_start_tag (self, node): +- if len(self._message) == 0 or not(isinstance(self._message[-1], basestring)): ++ if len(self._message) == 0 or not(isinstance(self._message[-1], string_types)): + self._message.append('') + if node.ns() is not None and node.ns().name is not None: +- self._message[-1] += (u'<%s:%s' % (unicode(node.ns().name, 'utf-8'), unicode(node.name, 'utf-8'))) ++ self._message[-1] += ('<%s:%s' % (ustr(node.ns().name, 'utf-8'), ustr(node.name, 'utf-8'))) + else: +- self._message[-1] += (u'<%s' % unicode(node.name, 'utf-8')) ++ self._message[-1] += ('<%s' % ustr(node.name, 'utf-8')) + for prop in xml_attr_iter(node): + name = prop.name + if prop.ns() is not None: + name = prop.ns().name + ':' + name + atval = prop.content +- if not isinstance(atval, unicode): +- atval = unicode(atval, 'utf-8') ++ if not isinstance(atval, ustr_type): ++ atval = ustr(atval, 'utf-8') + atval = atval.replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"') + self._message += " %s=\"%s\"" % (name, atval) + if node.children is not None: +@@ -244,12 +262,12 @@ class Message (object): + + def add_end_tag (self, node): + if node.children is not None: +- if len(self._message) == 0 or not(isinstance(self._message[-1], basestring)): ++ if len(self._message) == 0 or not(isinstance(self._message[-1], string_types)): + self._message.append('') + if node.ns() is not None and node.ns().name is not None: +- self._message[-1] += (u'</%s:%s>' % (unicode(node.ns().name, 'utf-8'), unicode(node.name, 'utf-8'))) ++ self._message[-1] += ('</%s:%s>' % (ustr(node.ns().name, 'utf-8'), ustr(node.name, 'utf-8'))) + else: +- self._message[-1] += (u'</%s>' % unicode(node.name, 'utf-8')) ++ self._message[-1] += ('</%s>' % ustr(node.name, 'utf-8')) + + def is_empty (self): + return self._empty +@@ -261,16 +279,16 @@ class Message (object): + self._ctxt = ctxt + + def add_source (self, source): +- if not isinstance(source, unicode): +- source = unicode(source, 'utf-8') ++ if not isinstance(source, ustr_type): ++ source = ustr(source, 'utf-8') + self._sources.append(source) + + def get_sources (self): + return self._sources + + def add_marker (self, marker): +- if not isinstance(marker, unicode): +- marker = unicode(marker, 'utf-8') ++ if not isinstance(marker, ustr_type): ++ marker = ustr(marker, 'utf-8') + self._markers.append(marker) + + def get_markers (self): +@@ -290,13 +308,13 @@ class Message (object): + return self._comments + + def get_string (self): +- message = u'' ++ message = '' + placeholder = 1 + for msg in self._message: +- if isinstance(msg, basestring): ++ if isinstance(msg, string_types): + message += msg + elif isinstance(msg, Message.Placeholder): +- message += u'<_:%s-%i/>' % (msg.name, placeholder) ++ message += '<_:%s-%i/>' % (msg.name, placeholder) + placeholder += 1 + if not self._preserve: + message = re.sub('\s+', ' ', message).strip() +@@ -315,10 +333,10 @@ class Message (object): + self._locale_filter = locale + + def format (self): +- ret = u'' ++ ret = '' + markers = {} + for marker in self._markers: +- if not markers.has_key(marker): ++ if marker not in markers: + ret += '#. (itstool) path: ' + marker + '\n' + markers[marker] = marker + for idvalue in self._id_values: +@@ -329,7 +347,7 @@ class Message (object): + commentsdict = {} + for comment in self._comments: + key = comment.get_text() +- if commentsdict.has_key(key): ++ if key in commentsdict: + for marker in comment.get_markers(): + commentsdict[key].add_marker(marker) + else: +@@ -340,23 +358,23 @@ class Message (object): + ret += '#.\n' + ret += comments[i].format() + for source in self._sources: +- ret += u'#: %s\n' % source ++ ret += '#: %s\n' % source + if self._preserve: +- ret += u'#, no-wrap\n' ++ ret += '#, no-wrap\n' + if self._ctxt is not None: +- ret += u'msgctxt "%s"\n' % self._ctxt ++ ret += 'msgctxt "%s"\n' % self._ctxt + message = self.get_string() + if self._preserve: +- ret += u'msgid ""\n' ++ ret += 'msgid ""\n' + lines = message.split('\n') +- for line, no in zip(lines, range(len(lines))): ++ for line, no in zip(lines, list(range(len(lines)))): + if no == len(lines) - 1: +- ret += u'"%s"\n' % self.escape(line) ++ ret += '"%s"\n' % self.escape(line) + else: +- ret += u'"%s\\n"\n' % self.escape(line) ++ ret += '"%s\\n"\n' % self.escape(line) + else: +- ret += u'msgid "%s"\n' % self.escape(message) +- ret += u'msgstr ""\n' ++ ret += 'msgid "%s"\n' % self.escape(message) ++ ret += 'msgstr ""\n' + return ret + + +@@ -413,7 +431,7 @@ def fix_node_ns (node, nsdefs): + nsdef = node.nsDefs() + while nsdef is not None: + nextnsdef = nsdef.next +- if nsdefs.has_key(nsdef.name) and nsdefs[nsdef.name] == nsdef.content: ++ if nsdef.name in nsdefs and nsdefs[nsdef.name] == nsdef.content: + node.removeNsDef(nsdef.content) + else: + childnsdefs[nsdef.name] = nsdef.content +@@ -508,7 +526,7 @@ class Document (object): + try: + self._check_errors() + except libxml2.parserError as e: +- sys.stderr.write('Error: Could not parse document:\n%s\n' % str(e)) ++ sys.stderr.write('Error: Could not parse document:\n%s\n' % ustr(e)) + sys.exit(1) + self._msgs = messages + self._its_translate_nodes = {} +@@ -532,7 +550,7 @@ class Document (object): + for child in xml_child_iter(rules): + if xml_is_ns_name(child, NS_ITS, 'param'): + name = child.nsProp('name', None) +- if params.has_key(name): ++ if name in params: + value = params[name] + else: + value = child.getContent() +@@ -586,7 +604,7 @@ class Document (object): + oldnode = None + xpath.setContextNode(node) + idvalue = self._try_xpath_eval(xpath, idv) +- if isinstance(idvalue, basestring): ++ if isinstance(idvalue, string_types): + self._its_id_values[node] = idvalue + else: + for val in idvalue: +@@ -606,7 +624,7 @@ class Document (object): + oldnode = None + xpath.setContextNode(node) + ctxt = self._try_xpath_eval(xpath, rule.nsProp('contextPointer', None)) +- if isinstance(ctxt, basestring): ++ if isinstance(ctxt, string_types): + self._itst_contexts[node] = ctxt + else: + for ctxt in ctxt: +@@ -642,7 +660,7 @@ class Document (object): + oldnode = None + xpath.setContextNode(node) + note = self._try_xpath_eval(xpath, sel) +- if isinstance(note, basestring): ++ if isinstance(note, string_types): + if ref: + nodenote = LocNote(locnoteref=note, locnotetype=notetype) + else: +@@ -751,7 +769,7 @@ class Document (object): + nsdef = par.nsDefs() + while nsdef is not None: + if nsdef.name is not None: +- if not nss.has_key(nsdef.name): ++ if nsdef.name not in nss: + nss[nsdef.name] = nsdef.content + xpath.xpathRegisterNs(nsdef.name, nsdef.content) + nsdef = nsdef.next +@@ -770,7 +788,7 @@ class Document (object): + nsdef = par.nsDefs() + while nsdef is not None: + if nsdef.name is not None: +- if not nss.has_key(nsdef.name): ++ if nsdef.name not in nss: + nss[nsdef.name] = nsdef.content + xpath.xpathRegisterNs(nsdef.name, nsdef.content) + nsdef = nsdef.next +@@ -875,7 +893,7 @@ class Document (object): + prevtext = node.prev.content + if re.sub('\s+', '', prevtext) == '': + prevnode = node.prev +- for lang in sorted(translations.keys(), reverse=True): ++ for lang in sorted(list(translations.keys()), reverse=True): + locale = self.get_its_locale_filter(node) + lmatch = match_locale_list(locale[0], lang) + if (locale[1] == 'include' and not lmatch) or (locale[1] == 'exclude' and lmatch): +@@ -966,7 +984,9 @@ class Document (object): + for attr in trans_attrs: + newcontent = translations.ugettext(attr.get_content()) + if newcontent: +- newnode.setProp(attr.name, translations.ugettext(attr.get_content())) ++ if not PY3: ++ newcontent = newcontent.encode('utf-8') ++ newnode.setProp(attr.name, newcontent) + + def get_translated (self, node, translations, strict=False, lang=None): + msg = self._msgs.get_message_by_node(node) +@@ -993,15 +1013,17 @@ class Document (object): + nss['_'] = NS_BLANK + try: + blurb = node.doc.intSubset().serialize('utf-8') +- except: ++ except Exception: + blurb = '' +- blurb += '<' + node.name +- for nsname in nss.keys(): ++ blurb += '<' + ustr(node.name, 'utf-8') ++ for nsname in list(nss.keys()): + if nsname is None: + blurb += ' xmlns="%s"' % nss[nsname] + else: + blurb += ' xmlns:%s="%s"' % (nsname, nss[nsname]) +- blurb += '>%s</%s>' % (trans.encode('utf-8'), node.name) ++ blurb += '>%s</%s>' % (trans, ustr(node.name, 'utf-8')) ++ if not PY3: ++ blurb = blurb.encode('utf-8') + ctxt = libxml2.createDocParserCtxt(blurb) + if self._load_dtd: + ctxt.loadSubset(1) +@@ -1010,7 +1032,7 @@ class Document (object): + trnode = ctxt.doc().getRootElement() + try: + self._check_errors() +- except libxml2.parserError as e: ++ except libxml2.parserError: + if strict: + raise + else: +@@ -1093,7 +1115,7 @@ class Document (object): + if self.get_its_locale_filter(node) != ('*', 'include'): + msg.set_locale_filter(self.get_its_locale_filter(node)) + msg.add_source('%s:%i' % (self._doc.name, node.lineNo())) +- msg.add_marker('%s/%s' % (node.parent.name, node.name)) ++ msg.add_marker('%s/%s' % (ustr(node.parent.name, 'utf-8'), ustr(node.name, 'utf-8'))) + else: + withinText = True + msg.add_start_tag(node) +@@ -1148,7 +1170,7 @@ class Document (object): + msg.add_end_tag(node) + + def generate_external_resource_message(self, node): +- if not self._its_externals.has_key(node): ++ if node not in self._its_externals: + return + resref = self._its_externals[node] + if node.type == 'element': +@@ -1162,10 +1184,10 @@ class Document (object): + msg = Message() + try: + fullfile = os.path.join(os.path.dirname(self._filename), resref) +- filefp = open(fullfile) ++ filefp = open(fullfile, 'rb') + filemd5 = hashlib.md5(filefp.read()).hexdigest() + filefp.close() +- except: ++ except Exception: + filemd5 = '__failed__' + txt = "external ref='%s' md5='%s'" % (resref, filemd5) + msg.set_context('_') +@@ -1189,7 +1211,7 @@ class Document (object): + while node.type in ('attribute', 'element'): + if node.getSpacePreserve() == 1: + return True +- if self._its_preserve_space_nodes.has_key(node): ++ if node in self._its_preserve_space_nodes: + return (self._its_preserve_space_nodes[node] == 'preserve') + node = node.parent + return False +@@ -1200,7 +1222,7 @@ class Document (object): + val = node.nsProp('translate', NS_ITS) + elif xml_is_ns_name(node, NS_ITS, 'span') and node.hasNsProp('translate', None): + val = node.nsProp('translate', None) +- elif self._its_translate_nodes.has_key(node): ++ elif node in self._its_translate_nodes: + val = self._its_translate_nodes[node] + if val is not None: + return val +@@ -1232,7 +1254,7 @@ class Document (object): + else: + typ = 'include' + return (lst, typ) +- if (xml_is_ns_name(node, NS_ITS, 'span') and ++ if (xml_is_ns_name(node, NS_ITS, 'span') and + (node.hasNsProp('localeFilterList', None) or node.hasNsProp('localeFilterType', None))): + if node.hasNsProp('localeFilterList', None): + lst = node.nsProp('localeFilterList', None) +@@ -1243,7 +1265,7 @@ class Document (object): + else: + typ = 'include' + return (lst, typ) +- if self._its_locale_filters.has_key(node): ++ if node in self._its_locale_filters: + return self._its_locale_filters[node] + if node.parent.type == 'element': + return self.get_its_locale_filter(node.parent) +@@ -1298,7 +1320,7 @@ class Document (object): + val = self.get_its_loc_notes(node) + if len(val) > 0: + if val[0].locnote is not None: +- compval = 'locNote="%s"\tlocNoteType="%s"' % (str(val[0]), val[0].locnotetype) ++ compval = 'locNote="%s"\tlocNoteType="%s"' % (ustr(val[0]), val[0].locnotetype) + elif val[0].locnoteref is not None: + compval = 'locNoteRef="%s"\tlocNoteType="%s"' % (val[0].locnoteref, val[0].locnotetype) + elif category == 'externalResourceRef': +@@ -1321,7 +1343,7 @@ class Document (object): + out.write('%s\t%s\r\n' % (xml_get_node_path(node), compval)) + else: + out.write('%s\r\n' % (xml_get_node_path(node))) +- for attr in sorted(xml_attr_iter(node), lambda x, y: cmp(str(x), str(y))): ++ for attr in sorted(xml_attr_iter(node), key=ustr): + self.output_test_data(category, out, attr) + for child in xml_child_iter(node): + if child.type == 'element': +@@ -1481,7 +1503,7 @@ if __name__ == '__main__': + out = sys.stdout + else: + try: +- out = file(opts.output, 'w') ++ out = io.open(opts.output, 'wt', encoding='utf-8') + except: + sys.stderr.write('Error: Cannot write to file %s\n' % opts.output) + sys.exit(1) +@@ -1495,6 +1517,8 @@ if __name__ == '__main__': + except: + sys.stderr.write('Error: cannot open mo file %s\n' % opts.merge) + sys.exit(1) ++ if PY3: ++ translations.ugettext = translations.gettext + translations.add_fallback(NoneTranslations()) + if opts.lang is None: + opts.lang = convert_locale(os.path.splitext(os.path.basename(opts.merge))[0]) +@@ -1506,7 +1530,7 @@ if __name__ == '__main__': + if opts.output == '-': + out = sys.stdout + else: +- out = file(opts.output, 'w') ++ out = open(opts.output, 'w') + else: + sys.stderr.write('Error: Non-directory output for multiple files\n') + sys.exit(1) +@@ -1520,11 +1544,11 @@ if __name__ == '__main__': + try: + doc.merge_translations(translations, opts.lang, strict=opts.strict) + except Exception as e: +- sys.stderr.write('Error: Could not merge translations:\n%s\n' % str(e)) ++ sys.stderr.write('Error: Could not merge translations:\n%s\n' % ustr(e)) + sys.exit(1) + fout = out +- if isinstance(fout, basestring): +- fout = file(os.path.join(fout, os.path.basename(filename)), 'w') ++ if isinstance(fout, string_types): ++ fout = open(os.path.join(fout, os.path.basename(filename)), 'w') + fout.write(doc._doc.serialize('utf-8')) + elif opts.join is not None: + translations = {} +@@ -1535,14 +1559,16 @@ if __name__ == '__main__': + sys.stderr.write('Error: cannot open mo file %s\n' % filename) + sys.exit(1) + thistr.add_fallback(NoneTranslations()) ++ if PY3: ++ thistr.ugettext = thistr.gettext + lang = convert_locale(os.path.splitext(os.path.basename(filename))[0]) + translations[lang] = thistr + if opts.output is None: + out = sys.stdout + elif os.path.isdir(opts.output): +- out = file(os.path.join(opts.output, os.path.basename(filename)), 'w') ++ out = open(os.path.join(opts.output, os.path.basename(filename)), 'w') + else: +- out = file(opts.output, 'w') ++ out = open(opts.output, 'w') + messages = MessageList() + doc = Document(opts.join, messages) + doc.apply_its_rules(not(opts.nobuiltins), params=params) +@@ -1555,9 +1581,9 @@ if __name__ == '__main__': + try: + doc.merge_translations(translations, opts.lang, strict=opts.strict) + except Exception as e: +- sys.stderr.write('Error: Could not merge translations:\n%s\n' % str(e)) ++ sys.stderr.write('Error: Could not merge translations:\n%s\n' % ustr(e)) + sys.exit(1) + fout = out +- if isinstance(fout, basestring): +- fout = file(os.path.join(fout, os.path.basename(filename)), 'w') ++ if isinstance(fout, string_types): ++ fout = open(os.path.join(fout, os.path.basename(filename)), 'w') + fout.write(doc._doc.serialize('utf-8')) + +From 89aff64b9e7fbfd74e586ef61c0804b646004e80 Mon Sep 17 00:00:00 2001 +From: Virgil Dupras <hsoft@hardcoded.net> +Date: Sat, 16 Jan 2016 16:05:54 -0500 +Subject: [PATCH 2/2] Fix problem with merging translations under py3 + +This fixes three problems at once: + +1. When fiddling with credits nodes, we would get an error about libxml2 +expecting `str` rather than `bytes`. We could fix this by encoding the +value only when uner py2. + +2. When writing the merged XML, our serlialized data would be `str` +under py3, which would cause implicit encoding problems when writing +that contents to the file. + +3. `fout` would not be closed after writing, which would sometimes cause +the target file to end up with no contents at all (at least on my +machine. +--- + itstool.in | 17 +++++++++++++---- + 1 file changed, 13 insertions(+), 4 deletions(-) + +diff --git a/itstool.in b/itstool.in +index 1a59a5a..45ff0c0 100755 +--- a/itstool.in ++++ b/itstool.in +@@ -839,7 +839,8 @@ class Document (object): + elif select == 'year' and len(trdata) == 4: + val = trdata[3] + if val is not None: +- val = val.encode('utf-8') ++ if not PY3: ++ val = val.encode('utf-8') + parent.addContent(val) + else: + newnode = node.copyNode(2) +@@ -1546,10 +1547,18 @@ if __name__ == '__main__': + except Exception as e: + sys.stderr.write('Error: Could not merge translations:\n%s\n' % ustr(e)) + sys.exit(1) ++ serialized = doc._doc.serialize('utf-8') ++ if PY3: ++ # For some reason, under py3, our serialized data is returns as a str. ++ # Let's encode it to bytes ++ serialized = serialized.encode('utf-8') + fout = out +- if isinstance(fout, string_types): +- fout = open(os.path.join(fout, os.path.basename(filename)), 'w') +- fout.write(doc._doc.serialize('utf-8')) ++ fout_is_str = isinstance(fout, string_types) ++ if fout_is_str: ++ fout = open(os.path.join(fout, os.path.basename(filename)), 'wb') ++ fout.write(serialized) ++ if fout_is_str: ++ fout.close() + elif opts.join is not None: + translations = {} + for filename in args[1:]: |