1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """Classes that hold units of .po files (pounit) or entire files (pofile).
23
24 Gettext-style .po (or .pot) files are used in translations for KDE, GNOME and
25 many other projects.
26
27 This uses libgettextpo from the gettext package. Any version before 0.17 will
28 at least cause some subtle bugs or may not work at all. Developers might want
29 to have a look at gettext-tools/libgettextpo/gettext-po.h from the gettext
30 package for the public API of the library.
31 """
32
33 from translate.misc.multistring import multistring
34 from translate.storage import pocommon
35 from translate.misc import quote
36 from translate.lang import data
37 from ctypes import *
38 import ctypes.util
39 try:
40 import cStringIO as StringIO
41 except ImportError:
42 import StringIO
43 import os
44 import pypo
45 import re
46 import sys
47
48 lsep = " "
49 """Seperator for #: entries"""
50
51 STRING = c_char_p
52
53
56
57
58 xerror_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING)
59 xerror2_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING)
60
61
62
64 _fields_ = [('xerror', xerror_prototype),
65 ('xerror2', xerror2_prototype)]
66
68 _fields_ = [
69 ('error', CFUNCTYPE(None, c_int, c_int, STRING)),
70 ('error_at_line', CFUNCTYPE(None, c_int, c_int, STRING, c_uint, STRING)),
71 ('multiline_warning', CFUNCTYPE(None, STRING, STRING)),
72 ('multiline_error', CFUNCTYPE(None, STRING, STRING)),
73 ]
74
75
76 -def xerror_cb(severity, message, filename, lineno, column, multilint_p, message_text):
77 print >> sys.stderr, "xerror_cb", severity, message, filename, lineno, column, multilint_p, message_text
78 if severity >= 1:
79 raise ValueError(message_text)
80
81 -def xerror2_cb(severity, message1, filename1, lineno1, column1, multiline_p1, message_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2):
82 print >> sys.stderr, "xerror2_cb", severity, message1, filename1, lineno1, column1, multiline_p1, message_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2
83 if severity >= 1:
84 raise ValueError(message_text1)
85
86
87
88
89 gpo = None
90
91
92 names = ['gettextpo', 'libgettextpo']
93 for name in names:
94 lib_location = ctypes.util.find_library(name)
95 if lib_location:
96 gpo = cdll.LoadLibrary(lib_location)
97 if gpo:
98 break
99 else:
100
101
102 try:
103 gpo = cdll.LoadLibrary('libgettextpo.so')
104 except OSError, e:
105 raise ImportError("gettext PO library not found")
106
107
108
109 gpo.po_file_read_v3.argtypes = [STRING, POINTER(po_xerror_handler)]
110 gpo.po_file_write_v2.argtypes = [c_int, STRING, POINTER(po_xerror_handler)]
111 gpo.po_file_write_v2.retype = c_int
112
113
114 gpo.po_file_domain_header.restype = STRING
115 gpo.po_header_field.restype = STRING
116 gpo.po_header_field.argtypes = [STRING, STRING]
117
118
119 gpo.po_filepos_file.restype = STRING
120 gpo.po_message_filepos.restype = c_int
121 gpo.po_message_filepos.argtypes = [c_int, c_int]
122 gpo.po_message_add_filepos.argtypes = [c_int, STRING, c_int]
123
124
125 gpo.po_message_comments.restype = STRING
126 gpo.po_message_extracted_comments.restype = STRING
127 gpo.po_message_prev_msgctxt.restype = STRING
128 gpo.po_message_prev_msgid.restype = STRING
129 gpo.po_message_prev_msgid_plural.restype = STRING
130 gpo.po_message_is_format.restype = c_int
131 gpo.po_message_msgctxt.restype = STRING
132 gpo.po_message_msgid.restype = STRING
133 gpo.po_message_msgid_plural.restype = STRING
134 gpo.po_message_msgstr.restype = STRING
135 gpo.po_message_msgstr_plural.restype = STRING
136
137
138 gpo.po_message_set_comments.argtypes = [c_int, STRING]
139 gpo.po_message_set_extracted_comments.argtypes = [c_int, STRING]
140 gpo.po_message_set_fuzzy.argtypes = [c_int, c_int]
141 gpo.po_message_set_msgctxt.argtypes = [c_int, STRING]
142
143
144 xerror_handler = po_xerror_handler()
145 xerror_handler.xerror = xerror_prototype(xerror_cb)
146 xerror_handler.xerror2 = xerror2_prototype(xerror2_cb)
147
150
153
156
159
160 -class pounit(pocommon.pounit):
181 msgid_plural = property(None, setmsgid_plural)
182
184 def remove_msgid_comments(text):
185 if not text:
186 return text
187 if text.startswith("_:"):
188 remainder = re.search(r"_: .*\n(.*)", text)
189 if remainder:
190 return remainder.group(1)
191 else:
192 return u""
193 else:
194 return text
195 singular = remove_msgid_comments(gpo.po_message_msgid(self._gpo_message))
196 if singular:
197 multi = multistring(singular, self._encoding)
198 if self.hasplural():
199 pluralform = gpo.po_message_msgid_plural(self._gpo_message)
200 if isinstance(pluralform, str):
201 pluralform = pluralform.decode(self._encoding)
202 multi.strings.append(pluralform)
203 return multi
204 else:
205 return u""
206
219
220 source = property(getsource, setsource)
221
223 if self.hasplural():
224 plurals = []
225 nplural = 0
226 plural = gpo.po_message_msgstr_plural(self._gpo_message, nplural)
227 while plural:
228 plurals.append(plural)
229 nplural += 1
230 plural = gpo.po_message_msgstr_plural(self._gpo_message, nplural)
231 if plurals:
232 multi = multistring(plurals, encoding=self._encoding)
233 else:
234 multi = multistring(u"")
235 else:
236 multi = multistring(gpo.po_message_msgstr(self._gpo_message) or u"", encoding=self._encoding)
237 return multi
238
240 if self.hasplural():
241 if isinstance(target, multistring):
242 target = target.strings
243 elif isinstance(target, basestring):
244 target = [target]
245 elif isinstance(target,(dict, list)):
246 if len(target) == 1:
247 target = target[0]
248 else:
249 raise ValueError("po msgid element has no plural but msgstr has %d elements (%s)" % (len(target), target))
250 if isinstance(target, (dict, list)):
251 i = 0
252 message = gpo.po_message_msgstr_plural(self._gpo_message, i)
253 while message is not None:
254 gpo.po_message_set_msgstr_plural(self._gpo_message, i, None)
255 i += 1
256 message = gpo.po_message_msgstr_plural(self._gpo_message, i)
257 if isinstance(target, list):
258 for i in range(len(target)):
259 targetstring = target[i]
260 if isinstance(targetstring, unicode):
261 targetstring = targetstring.encode(self._encoding)
262 gpo.po_message_set_msgstr_plural(self._gpo_message, i, targetstring)
263 elif isinstance(target, dict):
264 for i, targetstring in enumerate(target.itervalues()):
265 gpo.po_message_set_msgstr_plural(self._gpo_message, i, targetstring)
266 else:
267 if isinstance(target, unicode):
268 target = target.encode(self._encoding)
269 if target is None:
270 gpo.po_message_set_msgstr(self._gpo_message, "")
271 else:
272 gpo.po_message_set_msgstr(self._gpo_message, target)
273 target = property(gettarget, settarget)
274
276 """The unique identifier for this unit according to the convensions in
277 .mo files."""
278 id = gpo.po_message_msgid(self._gpo_message)
279
280
281
282
283
284
285
286 context = gpo.po_message_msgctxt(self._gpo_message)
287 if context:
288 id = "%s\04%s" % (context, id)
289 return id or ""
290
292 if origin == None:
293 comments = gpo.po_message_comments(self._gpo_message) + \
294 gpo.po_message_extracted_comments(self._gpo_message)
295 elif origin == "translator":
296 comments = gpo.po_message_comments(self._gpo_message)
297 elif origin in ["programmer", "developer", "source code"]:
298 comments = gpo.po_message_extracted_comments(self._gpo_message)
299 else:
300 raise ValueError("Comment type not valid")
301
302 if comments:
303 comments = "\n".join([line[1:] for line in comments.split("\n")])
304
305 return comments[:-1].decode(self._encoding)
306
307 - def addnote(self, text, origin=None, position="append"):
308 if not text:
309 return
310 text = data.forceunicode(text)
311 oldnotes = self.getnotes(origin)
312 newnotes = None
313 if oldnotes:
314 if position == "append":
315 newnotes = oldnotes + "\n" + text
316 elif position == "merge":
317 if oldnotes != text:
318 oldnoteslist = oldnotes.split("\n")
319 for newline in text.split("\n"):
320 newline = newline.rstrip()
321
322 if newline not in oldnotes or len(newline) < 5:
323 oldnoteslist.append(newline)
324 newnotes = "\n".join(oldnoteslist)
325 else:
326 newnotes = text + '\n' + oldnotes
327 else:
328 newnotes = "\n".join([line.rstrip() for line in text.split("\n")])
329
330 if newnotes:
331 newlines = []
332 for line in newnotes.split("\n"):
333 if line:
334 newlines.append(" " + line)
335 else:
336 newlines.append(line)
337 newnotes = "\n".join(newlines)
338 if origin in ["programmer", "developer", "source code"]:
339 gpo.po_message_set_extracted_comments(self._gpo_message, newnotes)
340 else:
341 gpo.po_message_set_comments(self._gpo_message, newnotes)
342
344 gpo.po_message_set_comments(self._gpo_message, "")
345
347 newpo = self.__class__()
348 newpo._gpo_message = self._gpo_message
349 return newpo
350
351 - def merge(self, otherpo, overwrite=False, comments=True, authoritative=False):
386
388
389
390 return self.getid() == "" and len(self.target) > 0
391
394
397
404
407
410
412 return gpo.po_message_is_fuzzy(self._gpo_message)
413
415 gpo.po_message_set_fuzzy(self._gpo_message, present)
416
419
421 return gpo.po_message_is_obsolete(self._gpo_message)
422
424
425
426 gpo.po_message_set_obsolete(self._gpo_message, True)
427
429 gpo.po_message_set_obsolete(self._gpo_message, False)
430
432 return gpo.po_message_msgid_plural(self._gpo_message) is not None
433
449
454
456 locations = []
457 i = 0
458 location = gpo.po_message_filepos(self._gpo_message, i)
459 while location:
460 locname = gpo.po_filepos_file(location)
461 locline = gpo.po_filepos_start_line(location)
462 if locline == -1:
463 locstring = locname
464 else:
465 locstring = locname + ":" + str(locline)
466 locations.append(locstring)
467 i += 1
468 location = gpo.po_message_filepos(self._gpo_message, i)
469 return locations
470
472 for loc in location.split():
473 parts = loc.split(":")
474 file = parts[0]
475 if len(parts) == 2:
476 line = int(parts[1])
477 else:
478 line = -1
479 gpo.po_message_add_filepos(self._gpo_message, file, line)
480
481 - def getcontext(self):
482 msgctxt = gpo.po_message_msgctxt(self._gpo_message)
483 msgidcomment = self._extract_msgidcomments()
484 if msgctxt:
485 return msgctxt + msgidcomment
486 else:
487 return msgidcomment
488
489 -class pofile(pocommon.pofile):
490 UnitClass = pounit
492 self.UnitClass = unitclass
493 pocommon.pofile.__init__(self, unitclass=unitclass)
494 self._gpo_memory_file = None
495 self._gpo_message_iterator = None
496 self._encoding = encodingToUse(encoding)
497 if inputfile is not None:
498 self.parse(inputfile)
499 else:
500 self._gpo_memory_file = gpo.po_file_create()
501 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_file, None)
502
504 gpo.po_message_insert(self._gpo_message_iterator, unit._gpo_message)
505 self.units.append(unit)
506
508 """make sure each msgid is unique ; merge comments etc from duplicates into original"""
509 msgiddict = {}
510 uniqueunits = []
511
512
513 markedpos = []
514 def addcomment(thepo):
515 thepo.msgidcomment = " ".join(thepo.getlocations())
516 markedpos.append(thepo)
517 for thepo in self.units:
518 if thepo.isheader():
519 uniqueunits.append(thepo)
520 continue
521 if duplicatestyle.startswith("msgid_comment"):
522 msgid = thepo._extract_msgidcomments() + thepo.source
523 else:
524 msgid = thepo.source
525 if duplicatestyle == "msgid_comment_all":
526 addcomment(thepo)
527 uniqueunits.append(thepo)
528 elif msgid in msgiddict:
529 if duplicatestyle == "merge":
530 if msgid:
531 msgiddict[msgid].merge(thepo)
532 else:
533 addcomment(thepo)
534 uniqueunits.append(thepo)
535 elif duplicatestyle == "keep":
536 uniqueunits.append(thepo)
537 elif duplicatestyle == "msgid_comment":
538 origpo = msgiddict[msgid]
539 if origpo not in markedpos:
540 addcomment(origpo)
541 addcomment(thepo)
542 uniqueunits.append(thepo)
543 elif duplicatestyle == "msgctxt":
544 origpo = msgiddict[msgid]
545 if origpo not in markedpos:
546 gpo.po_message_set_msgctxt(origpo._gpo_message, " ".join(origpo.getlocations()))
547 markedpos.append(thepo)
548 gpo.po_message_set_msgctxt(thepo._gpo_message, " ".join(thepo.getlocations()))
549 uniqueunits.append(thepo)
550 else:
551 if not msgid and duplicatestyle != "keep":
552 addcomment(thepo)
553 msgiddict[msgid] = thepo
554 uniqueunits.append(thepo)
555 new_gpo_memory_file = gpo.po_file_create()
556 new_gpo_message_iterator = gpo.po_message_iterator(new_gpo_memory_file, None)
557 for unit in uniqueunits:
558 gpo.po_message_insert(new_gpo_message_iterator, unit._gpo_message)
559 gpo.po_message_iterator_free(self._gpo_message_iterator)
560 self._gpo_message_iterator = new_gpo_message_iterator
561 self._gpo_memory_file = new_gpo_memory_file
562 self.units = uniqueunits
563
565 def obsolete_workaround():
566
567
568
569 for unit in self.units:
570 if unit.isobsolete():
571 gpo.po_message_set_extracted_comments(unit._gpo_message, "")
572 location = gpo.po_message_filepos(unit._gpo_message, 0)
573 while location:
574 gpo.po_message_remove_filepos(unit._gpo_message, 0)
575 location = gpo.po_message_filepos(unit._gpo_message, 0)
576 outputstring = ""
577 if self._gpo_memory_file:
578 obsolete_workaround()
579 outputfile = os.tmpnam()
580 f = open(outputfile, "w")
581 self._gpo_memory_file = gpo.po_file_write_v2(self._gpo_memory_file, outputfile, xerror_handler)
582 f.close()
583 f = open(outputfile, "r")
584 outputstring = f.read()
585 f.close()
586 os.remove(outputfile)
587 return outputstring
588
590 """Returns True if the object doesn't contain any translation units."""
591 if len(self.units) == 0:
592 return True
593
594 if self.units[0].isheader():
595 units = self.units[1:]
596 else:
597 units = self.units
598
599 for unit in units:
600 if not unit.isblank() and not unit.isobsolete():
601 return False
602 return True
603
605 if hasattr(input, 'name'):
606 self.filename = input.name
607 elif not getattr(self, 'filename', ''):
608 self.filename = ''
609 if hasattr(input, "read"):
610 posrc = input.read()
611 input.close()
612 input = posrc
613 needtmpfile = not os.path.isfile(input)
614 if needtmpfile:
615
616 tmpfile = os.tmpnam()
617 f = open(tmpfile, "w")
618 f.write(input)
619 f.close()
620 input = tmpfile
621 self._gpo_memory_file = gpo.po_file_read_v3(input, xerror_handler)
622 if self._gpo_memory_file is None:
623 print >> sys.stderr, "Error:"
624 if needtmpfile:
625 os.remove(tmpfile)
626
627 self._header = gpo.po_file_domain_header(self._gpo_memory_file, None)
628 if self._header:
629 charset = gpo.po_header_field(self._header, "Content-Type")
630 if charset:
631 charset = re.search("charset=([^\\s]+)", charset).group(1)
632 self._encoding = encodingToUse(charset)
633 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_file, None)
634 newmessage = gpo.po_next_message(self._gpo_message_iterator)
635 while newmessage:
636 newunit = pounit(gpo_message=newmessage)
637 self.units.append(newunit)
638 newmessage = gpo.po_next_message(self._gpo_message_iterator)
639 self._free_iterator()
640
642
643
644 return
645 self._free_iterator()
646 if self._gpo_memory_file is not None:
647 gpo.po_file_free(self._gpo_memory_file)
648 self._gpo_memory_file = None
649
651
652
653 return
654 if self._gpo_message_iterator is not None:
655 gpo.po_message_iterator_free(self._gpo_message_iterator)
656 self._gpo_message_iterator = None
657