xref: /AOO41X/main/l10ntools/inc/export.hxx (revision ca62e2c2083b5d0995f1245bad6c2edfb455fbec) 
1 /**************************************************************
2  *
3  * Licensed to the Apache Software Foundation (ASF) under one
4  * or more contributor license agreements.  See the NOTICE file
5  * distributed with this work for additional information
6  * regarding copyright ownership.  The ASF licenses this file
7  * to you under the Apache License, Version 2.0 (the
8  * "License"); you may not use this file except in compliance
9  * with the License.  You may obtain a copy of the License at
10  *
11  *   http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing,
14  * software distributed under the License is distributed on an
15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16  * KIND, either express or implied.  See the License for the
17  * specific language governing permissions and limitations
18  * under the License.
19  *
20  *************************************************************/
21 
22 
23 
24 #ifndef _EXPORT_HXX
25 #define _EXPORT_HXX
26 
27 #include <l10ntools/directory.hxx>
28 
29 #include <tagtest.hxx>
30 
31 // #define MERGE_SOURCE_LANGUAGES <- To merge en-US and de resource
32 
33 #include <tools/string.hxx>
34 #include <tools/list.hxx>
35 #include <tools/stream.hxx>
36 #include <tools/fsys.hxx>
37 #include <osl/file.hxx>
38 #include <osl/file.h>
39 
40 #include <hash_map> /* std::hashmap*/
41 #include <iterator> /* std::iterator*/
42 #include <set>      /* std::set*/
43 #include <vector>   /* std::vector*/
44 #include <queue>
45 #include <string>
46 
47 #include <unistd.h>
48 #ifdef WNT
49 #include <direct.h>
50 #endif
51 
52 #define NO_TRANSLATE_ISO        "x-no-translate"
53 
54 #define JAPANESE_ISO "ja"
55 
56 class PFormEntrys;
57 class MergeData;
58 typedef std::set<ByteString , lessByteString > ByteStringSet;
59 
60 typedef std::hash_map<ByteString , ByteString , hashByteString,equalByteString>
61                                 ByteStringHashMap;
62 
63 typedef std::hash_map<ByteString , bool , hashByteString,equalByteString>
64                                 ByteStringBoolHashMap;
65 
66 typedef std::hash_map<ByteString , PFormEntrys* , hashByteString,equalByteString>
67                                 PFormEntrysHashMap;
68 
69 typedef std::hash_map<ByteString , MergeData* , hashByteString,equalByteString>
70                                 MergeDataHashMap;
71 
72 #define SOURCE_LANGUAGE ByteString("en-US")
73 #define LIST_REFID  "LIST_REFID"
74 
75 typedef ByteStringHashMap ExportListEntry;
76 
77 DECLARE_LIST( ExportListBase, ExportListEntry * )
78 
79 //
80 // class ExportList
81 //
82 
83 class ExportList : public ExportListBase
84 {
85 private:
86     sal_uLong nSourceLanguageListEntryCount;
87 
88 public:
ExportList()89     ExportList() : ExportListBase() { nSourceLanguageListEntryCount = 0; }
GetSourceLanguageListEntryCount()90     sal_uLong GetSourceLanguageListEntryCount() { return nSourceLanguageListEntryCount; }
NewSourceLanguageListEntry()91     void NewSourceLanguageListEntry() { nSourceLanguageListEntryCount++; }
92 };
93 
94 #define REFID_NONE 0xFFFF
95 
96 //
97 // struct ResData
98 //
99 
100 /******************************************************************************
101 * Purpose: holds mandatory data to export a single res (used with ResStack)
102 ******************************************************************************/
103 
104 #define ID_LEVEL_NULL       0x0000
105 #define ID_LEVEL_AUTOID     0x0001
106 #define ID_LEVEL_TEXT       0x0002
107 #define ID_LEVEL_FIELDNAME  0x0003
108 #define ID_LEVEL_ACCESSPATH 0x0004
109 #define ID_LEVEL_IDENTIFIER 0x0005
110 #define ID_LEVEL_LISTINDEX  0x0006
111 
112 class ResData
113 {
114 public:
115     ~ResData();
116     sal_Bool SetId( const ByteString &rId, sal_uInt16 nLevel );
117 
118     sal_uInt16 nWidth;
119     sal_uInt16 nChildIndex;
120     sal_uInt16 nIdLevel;
121     sal_Bool bChild;
122     sal_Bool bChildWithText;
123 
124     sal_Bool bText;
125     sal_Bool bHelpText;
126     sal_Bool bQuickHelpText;
127     sal_Bool bTitle;
128     sal_Bool bList;
129 
130     sal_Bool bRestMerged;
131 
132     ByteString sResTyp;
133     ByteString sId;
134     ByteString sGId;
135     ByteString sHelpId;
136     ByteString sFilename;
137 
138     ByteStringHashMap sText;
139     sal_uInt16 nTextRefId;
140 
141     ByteStringHashMap sHelpText;
142     sal_uInt16 nHelpTextRefId;
143 
144     ByteStringHashMap sQuickHelpText;
145     sal_uInt16 nQuickHelpTextRefId;
146 
147     ByteStringHashMap sTitle;
148     sal_uInt16 nTitleRefId;
149 
150     ByteString sTextTyp;
151     ByteStringHashMap aFallbackData;
152     ByteStringHashMap aMergedLanguages;
153 
154     ExportList  *pStringList;
155     ExportList  *pUIEntries;
156     ExportList  *pItemList;
157     ExportList  *pFilterList;
158     ExportList  *pPairedList;
159 
160     ByteString sPForm;
161 
162     void Dump();
163     void addFallbackData( ByteString& sId , const ByteString& sText );
164     bool getFallbackData( ByteString& sId , ByteString& sText);
165 
166     void addMergedLanguage( ByteString& sLang );
167     bool isMerged( ByteString& sLang );
ResData(const ByteString & rPF,const ByteString & rGId)168     ResData( const ByteString &rPF, const ByteString &rGId )
169             :
170             nWidth( 0 ),
171             nChildIndex( 0 ),
172             nIdLevel( ID_LEVEL_NULL ),
173             bChild( sal_False ),
174             bChildWithText( sal_False ),
175             bText( sal_False ),
176             bHelpText( sal_False ),
177             bQuickHelpText( sal_False ),
178             bTitle( sal_False ),
179             bList( sal_False ),
180             bRestMerged( sal_False ),
181             sGId( rGId ),
182             nTextRefId( REFID_NONE ),
183             nHelpTextRefId( REFID_NONE ),
184             nQuickHelpTextRefId( REFID_NONE ),
185             nTitleRefId( REFID_NONE ),
186             sTextTyp( "Text" ),
187             pStringList( NULL ),
188             pUIEntries( NULL ),
189             pItemList( NULL ),
190             pFilterList( NULL ),
191             pPairedList( NULL ),
192             sPForm( rPF )
193     {
194         sGId.EraseAllChars( '\r' );
195         sPForm.EraseAllChars( '\r' );
196     };
ResData(const ByteString & rPF,const ByteString & rGId,const ByteString & rFilename)197     ResData( const ByteString &rPF, const ByteString &rGId , const ByteString &rFilename )
198             :
199             nChildIndex( 0 ),
200             nIdLevel( ID_LEVEL_NULL ),
201             bChild( sal_False ),
202             bChildWithText( sal_False ),
203             bText( sal_False ),
204             bHelpText( sal_False ),
205             bQuickHelpText( sal_False ),
206             bTitle( sal_False ),
207             bList( sal_False ),
208             bRestMerged( sal_False ),
209             sGId( rGId ),
210             sFilename( rFilename ),
211             nTextRefId( REFID_NONE ),
212             nHelpTextRefId( REFID_NONE ),
213             nQuickHelpTextRefId( REFID_NONE ),
214             nTitleRefId( REFID_NONE ),
215             sTextTyp( "Text" ),
216             pStringList( NULL ),
217             pUIEntries( NULL ),
218             pItemList( NULL ),
219             pFilterList( NULL ),
220             pPairedList( NULL ),
221             sPForm( rPF )
222 
223     {
224         sGId.EraseAllChars( '\r' );
225         sPForm.EraseAllChars( '\r' );
226     };
227 
228 
229 };
230 
231 
232 //
233 // class Export
234 //
235 
236 /******************************************************************************
237 * Purpose: syntax check and export of *.src, called from lexer
238 ******************************************************************************/
239 
240 #define LIST_NON                    0x0000
241 #define LIST_STRING                 0x0001
242 #define LIST_FILTER                 0x0002
243 #define LIST_ITEM                   0x0004
244 #define LIST_PAIRED                 0x0005
245 #define LIST_UIENTRIES              0x0008
246 #define STRING_TYP_TEXT             0x0010
247 #define STRING_TYP_HELPTEXT         0x0020
248 #define STRING_TYP_QUICKHELPTEXT    0x0040
249 #define STRING_TYP_TITLE            0x0080
250 
251 #define MERGE_MODE_NORMAL           0x0000
252 #define MERGE_MODE_LIST             0x0001
253 
254 DECLARE_LIST( ResStack, ResData * )
255 // forwards
256 class WordTransformer;
257 class ParserQueue;
258 
259 class Export
260 {
261 private:
262     WordTransformer *pWordTransformer;
263 
264     CharSet aCharSet;                   // used charset in src
265 
266     SvFileStream aOutput;
267 
268     ResStack aResStack;                 // stack for parsing recursive
269 
270     ByteString sActPForm;               // hold cur. system
271 
272     sal_Bool bDefine;                       // cur. res. in a define?
273     sal_Bool bNextMustBeDefineEOL;          // define but no \ at lineend
274     sal_uLong nLevel;                       // res. recursiv? how deep?
275     sal_uInt16 nList;                       // cur. res. is String- or FilterList
276     ByteString nListLang;
277     sal_uLong nListIndex;
278     sal_uLong nListLevel;
279     bool bSkipFile;
280     ByteString sProject;
281     ByteString sRoot;
282     sal_Bool bEnableExport;
283     sal_Bool bMergeMode;
284     ByteString sMergeSrc;
285     ByteString sLastListLine;
286     sal_Bool bError;                        // any errors while export?
287     sal_Bool bReadOver;
288     sal_Bool bDontWriteOutput;
289     ByteString sLastTextTyp;
290     static bool isInitialized;
291     ByteString sFilename;
292 
293 
294 public:
295     ParserQueue* pParseQueue; // public ?
296     static ByteString sLanguages; // public ?
297     static ByteString sForcedLanguages; // public ?
298 
299 
300     static bool skipProject( ByteString sPrj ) ;
301     static void InitLanguages( bool bMergeMode = false );
302     static void InitForcedLanguages( bool bMergeMode = false );
303     static std::vector<ByteString> GetLanguages();
304     static std::vector<ByteString> GetForcedLanguages();
305 
306     static void SetLanguages( std::vector<ByteString> val );
307     static void RemoveUTF8ByteOrderMarker( ByteString &rString );
308     static bool hasUTF8ByteOrderMarker( const ByteString &rString );
309     static void RemoveUTF8ByteOrderMarkerFromFile( const ByteString &rFilename );
310     static bool fileHasUTF8ByteOrderMarker( const ByteString &rString );
311     static ByteString GetIsoLangByIndex( sal_uInt16 nIndex );
312     static void QuotHTML( ByteString &rString );
313     static bool CopyFile( const ByteString& source , const ByteString& dest );
314 
315     static void QuotHTMLXRM( ByteString &rString );
316     static void UnquotHTML( ByteString &rString );
317 
318     static const char* GetEnv( const char *pVar );
319     static int getCurrentDirectory( rtl::OUString& base_fqurl , rtl::OUString& base );
320 
321     static bool isSourceLanguage( const ByteString &sLanguage );
322     static bool isAllowed( const ByteString &sLanguage );
323 
324     static bool LanguageAllowed( const ByteString &nLanguage );
325     static void Languages( std::vector<ByteString>::const_iterator& begin , std::vector<ByteString>::const_iterator& end );
326     static void getRandomName( const ByteString& sPrefix , ByteString& sRandStr , const ByteString& sPostfix  );
327     static void getRandomName( ByteString& sRandStr );
328     static void getCurrentDir( std::string& dir );
329 
330     static void replaceEncoding( ByteString& rString );
331 
332     static ByteString GetFallbackLanguage( const ByteString nLanguage );
333     static void FillInFallbacks( ResData *pResData );
334     static void FillInListFallbacks( ExportList *pList, const ByteString &nSource, const ByteString &nFallback );
335     static ByteString GetTimeStamp();
336     static sal_Bool ConvertLineEnds( ByteString sSource, ByteString sDestination );
337     static ByteString GetNativeFile( ByteString sSource );
338     static DirEntry GetTempFile();
339 
340     static void DumpExportList( ByteString& sListName , ExportList& aList );
341     static ByteString DumpMap( ByteString& sMapName , ByteStringHashMap& aMap );
342 
343 private:
344     static std::vector<ByteString> aLanguages;
345     static std::vector<ByteString> aForcedLanguages;
346 
347     sal_Bool ListExists( ResData *pResData, sal_uInt16 nLst );
348 
349     sal_Bool WriteData( ResData *pResData, sal_Bool bCreateNew = sal_False );// called befor dest. cur ResData
350     sal_Bool WriteExportList( ResData *pResData, ExportList *pExportList,
351                         const ByteString &rTyp, sal_Bool bCreateNew = sal_False );
352 
353     ByteString MergePairedList( ByteString& sLine , ByteString& sText );
354 
355     ByteString FullId();                    // creates cur. GID
356 
357     bool PairedListFallback( ByteString& sText , ResData& aResData );
358 
359     ByteString GetPairedListID      ( const ByteString& sText );
360     ByteString GetPairedListString  ( const ByteString& sText );
361     ByteString StripList    ( const ByteString& sText );
362 
363     void UnmergeUTF8( ByteString& sOrig );
364     void InsertListEntry( const ByteString &rText, const ByteString &rLine );
365     void CleanValue( ByteString &rValue );
366     ByteString GetText( const ByteString &rSource, int nToken );
367 
368     sal_Bool PrepareTextToMerge( ByteString &rText, sal_uInt16 nTyp,
369         ByteString &nLangIndex, ResData *pResData );
370 
371     void MergeRest( ResData *pResData, sal_uInt16 nMode = MERGE_MODE_NORMAL );
372     void ConvertMergeContent( ByteString &rText );
373 
374     void WriteToMerged( const ByteString &rText , bool bSDFContent );
375     void SetChildWithText();
376 
377     void CutComment( ByteString &rText );
378 
379 public:
380     Export( const ByteString &rOutput, sal_Bool bWrite,
381             const ByteString &rPrj, const ByteString &rPrjRoot , const ByteString& rFile );
382     Export( const ByteString &rOutput, sal_Bool bWrite,
383             const ByteString &rPrj, const ByteString &rPrjRoot,
384             const ByteString &rMergeSource , const ByteString& rFile );
385     ~Export();
386 
387     void Init();
388     int Execute( int nToken, const char * pToken ); // called from lexer
SetError()389     void SetError() { bError = sal_True; }
GetError()390     sal_Bool GetError() { return bError; }
391 };
392 
393 
394 //
395 // class PFormEntrys
396 //
397 
398 /******************************************************************************
399 * Purpose: holds information of data to merge (one pform)
400 ******************************************************************************/
401 
402 class PFormEntrys : public ByteString
403 {
404 friend class MergeDataFile;
405 private:
406     ByteString sHelpText; // empty string
407     ByteStringHashMap sText;
408     ByteStringBoolHashMap bTextFirst;
409     ByteStringHashMap sQuickHelpText;
410     ByteStringBoolHashMap bQuickHelpTextFirst;
411     ByteStringHashMap sTitle;
412     ByteStringBoolHashMap bTitleFirst;
413 
414 public:
PFormEntrys(const ByteString & rPForm)415     PFormEntrys( const ByteString &rPForm ) : ByteString( rPForm ) {};
416     ByteString Dump();
InsertEntry(const ByteString & nId,const ByteString & rText,const ByteString & rQuickHelpText,const ByteString & rTitle)417     void InsertEntry(
418                     const ByteString &nId ,
419                     const ByteString &rText,
420                     const ByteString &rQuickHelpText,
421                     const ByteString &rTitle
422                     )
423         {
424             sText[ nId ] = rText;
425             bTextFirst[ nId ] = true;
426             sQuickHelpText[ nId ] = rQuickHelpText;
427             bQuickHelpTextFirst[ nId ] = true;
428             sTitle[ nId ] = rTitle;
429             bTitleFirst[ nId ] = true;
430         }
431      sal_Bool GetText( ByteString &rReturn, sal_uInt16 nTyp, const ByteString &nLangIndex, sal_Bool bDel = sal_False );
432      sal_Bool GetTransex3Text( ByteString &rReturn, sal_uInt16 nTyp, const ByteString &nLangIndex, sal_Bool bDel = sal_False );
433 
434 };
435 
436 //
437 // class MergeData
438 //
439 
440 /******************************************************************************
441 * Purpose: holds information of data to merge (one ressource)
442 ******************************************************************************/
443 
444 class MergeDataFile;
445 
446 class MergeData
447 {
448 friend class MergeDataFile;
449 private:
450     ByteString sTyp;
451     ByteString sGID;
452     ByteString sLID;
453     ByteString sFilename;
454     PFormEntrysHashMap aMap;
455 public:
MergeData(const ByteString & rTyp,const ByteString & rGID,const ByteString & rLID,const ByteString & rFilename)456     MergeData( const ByteString &rTyp, const ByteString &rGID, const ByteString &rLID , const ByteString &rFilename )
457             : sTyp( rTyp ), sGID( rGID ), sLID( rLID ) , sFilename( rFilename ) {};
458     ~MergeData();
459     PFormEntrys* InsertEntry( const ByteString &rPForm );
460     PFormEntrys* GetPFormEntrys( ResData *pResData );
461 
462     void Insert( const ByteString& rPFO , PFormEntrys* pfEntrys );
463     PFormEntrys* GetPFObject( const ByteString& rPFO );
464 
465     ByteString Dump();
466     sal_Bool operator==( ResData *pData );
467 };
468 
469 //
470 // class MergeDataFile
471 //
472 
473 /******************************************************************************
474 * Purpose: holds information of data to merge
475 ******************************************************************************/
476 
477 class MergeDataFile
478 {
479     private:
480         sal_Bool bErrorLog;
481         ByteString sErrorLog;
482         SvFileStream aErrLog;
483         MergeDataHashMap aMap;
484         std::set<ByteString> aLanguageSet;
485 
486         MergeData *GetMergeData( ResData *pResData , bool bCaseSensitve = false );
487         void InsertEntry( const ByteString &rTYP, const ByteString &rGID, const ByteString &rLID,
488             const ByteString &rPFO,
489             const ByteString &nLang, const ByteString &rTEXT,
490             const ByteString &rQHTEXT, const ByteString &rTITLE,
491             const ByteString &sFilename, bool bCaseSensitive
492             );
493         ByteString Dump();
494         void WriteError( const ByteString &rLine );
495 
496     public:
497         MergeDataFile( const ByteString &rFileName, const ByteString& rFile , sal_Bool bErrLog, CharSet aCharSet, bool bCaseSensitive = false );
498         ~MergeDataFile();
499 
500         std::vector<ByteString> GetLanguages();
501 
502         PFormEntrys *GetPFormEntrys( ResData *pResData );
503         PFormEntrys *GetPFormEntrysCaseSensitive( ResData *pResData );
504 
505         static ByteString CreateKey( const ByteString& rTYP , const ByteString& rGID , const ByteString& rLID , const ByteString& rFilename , bool bCaseSensitive = false );
506 };
507 
508 
509 class QueueEntry
510 {
511 public:
QueueEntry(int nTypVal,ByteString sLineVal)512     QueueEntry( int nTypVal , ByteString sLineVal ): nTyp( nTypVal ) , sLine( sLineVal ){};
513     int nTyp;
514     ByteString sLine;
515 };
516 
517 class ParserQueue
518 {
519 public:
520 
521     ParserQueue( Export& aExportObj );
522     ~ParserQueue();
523 
524     inline void Push( const QueueEntry& aEntry );
525     bool bCurrentIsM;  // public ?
526     bool bNextIsM;   // public ?
527     bool bLastWasM;   // public ?
528     bool bMflag;   // public ?
529 
530     void Close();
531 private:
532     // Future / Next
533     std::queue<QueueEntry>* aQueueNext;
534     // Current
535     std::queue<QueueEntry>* aQueueCur;
536     // Ref
537     std::queue<QueueEntry>* aQref;
538 
539     Export& aExport;
540     bool bStart;
541     bool bStartNext;
542 
543     inline void Pop( std::queue<QueueEntry>& aQueue );
544 
545 };
546 #endif
547 
548