creaImageIO_lib
creaImageIOTreeHandlerImageAdder.cpp
Go to the documentation of this file.
1 /*
2 # ---------------------------------------------------------------------
3 #
4 # Copyright (c) CREATIS (Centre de Recherche en Acquisition et Traitement de l'Image
5 # pour la Santé)
6 # Authors : Eduardo Davila, Frederic Cervenansky, Claire Mouton
7 # Previous Authors : Laurent Guigues, Jean-Pierre Roux
8 # CreaTools website : www.creatis.insa-lyon.fr/site/fr/creatools_accueil
9 #
10 # This software is governed by the CeCILL-B license under French law and
11 # abiding by the rules of distribution of free software. You can use,
12 # modify and/ or redistribute the software under the terms of the CeCILL-B
13 # license as circulated by CEA, CNRS and INRIA at the following URL
14 # http://www.cecill.info/licences/Licence_CeCILL-B_V1-en.html
15 # or in the file LICENSE.txt.
16 #
17 # As a counterpart to the access to the source code and rights to copy,
18 # modify and redistribute granted by the license, users are provided only
19 # with a limited warranty and the software's author, the holder of the
20 # economic rights, and the successive licensors have only limited
21 # liability.
22 #
23 # The fact that you are presently reading this means that you have had
24 # knowledge of the CeCILL-B license and that you accept its terms.
25 # ------------------------------------------------------------------------
26 */
27 
29 #include <creaImageIOSystem.h>
30 #include <boost/filesystem.hpp>
31 #include <boost/algorithm/string.hpp>
32 #include <boost/utility.hpp>
33 
34 
35 namespace fs = boost::filesystem;
36 using boost::filesystem::path;
37 using boost::next;
38 using boost::prior;
39 
40 
41 using namespace crea;
42 using namespace boost;
43 
44 namespace creaImageIO
45 {
46  //====================================================================
47  // Ctor
48  TreeHandlerImageAdder::TreeHandlerImageAdder(TreeHandler* tree)
49  : mTreeHandler(tree)
50  {
51  }
52  // Dtor
54  {
55  }
56  //====================================================================
57 
58  //====================================================================
60  {
61  mProgressSignal.connect(callback);
62  }
63  //====================================================================
64 
65  //=====================================================================
66  bool TreeHandlerImageAdder::IsHandledFile( const std::string& filename)
67  {
68  return (mReader.CanRead(filename));
69  }
70  //=====================================================================
71 
72  //=====================================================================
73  void TreeHandlerImageAdder::AddFiles( const std::vector<std::string>& filenames)
74  {
75  mProgress.Reset();
76 
77  //unsigned int nbf = filenames.size();
78  std::vector<std::string>::const_iterator i;
80  for (i=filenames.begin();i!=filenames.end();++i)
81  {
82 
84  if (IsHandledFile(*i))
85  {
87  if(mSynchronizer->isIndexed(*i))
88  {
89  mSynchronizer->InsertAddOp((*i),"0","1",mCurrentDB);
90  std::string addKey=mSynchronizer->GetAttribute("ADD_KEY","ADD_OPS","PATH",(*i),mCurrentDB);
91  std::stringstream removedOn;
92  removedOn<<time(0);
93  mSynchronizer->InsertIgnoreFile(addKey,(*i),"0",removedOn.str(),mCurrentDB);
94  AddFile(*i);
95  }
96  }
98  if (mProgress.GetStop()) break;
99  }
100  }
101  //=====================================================================
102 
103  //=====================================================================
104  void TreeHandlerImageAdder::AddDirectory( const std::string& directory,
105  bool recurse)
106  {
107  mProgress.Reset();
108  std::stringstream files;
109 
110  std::stringstream rec;
111  rec<<recurse;
112  mSynchronizer->InsertAddOp(directory,rec.str(),"0",mCurrentDB);
113  std::string addKey=mSynchronizer->GetAttribute("ADD_KEY","ADD_OPS","PATH",directory,mCurrentDB);
116  AddDirectoryRecursor( directory, recurse, addKey );
117  //DicomImageScanner sc;
118  //AddDirectoryRecursorScanner(directory, recurse, addKey, sc, false );
119 
120  int nFiles=GetProgress().GetNumberAddedFiles();
121  files<<nFiles;
122  mSynchronizer->SetAttribute("FILES_ADDED","ADD_OPS",files.str(),"ADD_KEY",addKey,mCurrentDB);
124  GimmickDebugMessage(3,mProgress<<std::endl);
125  }
126 
127  //=====================================================================
128  void TreeHandlerImageAdder::AddFile( const std::string& filename )
129  {
130  GimmickDebugMessage(4,"Adding '"<<filename<<"'"<<std::endl);
131  std::map< std::string, std::string> attr;
133 
134 
135  mReader.ReadAttributes(filename,attr);
138  //bool bSOPIID = false;
139  //std::map<std::string, std::string>::iterator it_att = attr.begin();
140  //for(; it_att != attr.end(); it_att++)
141  //{
142  // if (it_att->first == "D0008_0018")
143  // {
144  // bSOPIID = mTreeHandler->TestSOPIID(it_attr->second);
145  // break;
146  // }
147  //}
148  //if(bSOPIID)
149  // return;
150 
151  int lev = mTreeHandler->AddBranch(attr);
152 
153  // update the progress according to lev
154  if (lev<mTreeHandler->GetTree().GetNumberOfLevels())
156  }
157  //=====================================================================
158 
160  {
161  int n=node->GetNumberOfChildren();
162  if(n>0)
163  {
164  RemoveFiles(node->GetChildrenList());
165  }
166  else
167  {
168  remove(node);
169  }
170  }
171 
172  //=====================================================================
173 
174  void TreeHandlerImageAdder::RemoveFiles(const std::vector<tree::Node*>& nodes)
175  {
176  std::vector<tree::Node*>::const_iterator it;
177  for(it=nodes.begin();it!=nodes.end();++it)
178  {
179  int n=(*it)->GetNumberOfChildren();
180  if(n>0)
181  {
182  RemoveFiles((*it)->GetChildrenList());
183  }
184  else
185  {
186  remove(*it);
187  }
188 
189  }
190  }
191 
192 
194  {
195  std::string path=i_node->GetAttribute("FullFileName");
196  //Gets the add key
197  std::string addKey=mSynchronizer->GetAttribute("ADD_KEY","IGNORED_FILES","PATH",path,mCurrentDB);
198  //Gets the number of files added
199  int files=atoi((mSynchronizer->GetAttribute("FILES_ADDED","ADD_OPS","ADD_KEY",addKey,mCurrentDB)).c_str());
200  files=files-1;
201  std::stringstream out;
202  out<<files;
203  //Sets the new number of files
204  mSynchronizer->SetAttribute("FILES_ADDED","ADD_OPS",out.str(),"ADD_KEY",addKey,mCurrentDB);
205  //Sets the file as removed
206  mSynchronizer->SetAttribute("REMOVE","IGNORED_FILES","1","PATH = '"+path+"' AND ADD_KEY",addKey,mCurrentDB);
207  }
208 
209 
210  //=======================================================================
211  //=====================================================================
212 #if defined(USE_GDCM2)
213 
214  void TreeHandlerImageAdder::AddDirectoryRecursorScanner(const std::string &dirpath,
215  bool recursive,const std::string &addKey, DicomImageScanner i_sc, bool b_loaded)
216  {
217  GimmickDebugMessage(4,"Scanning '"<<dirpath<<"'"<<std::endl);
219 
220  if ( !fs::exists( dirpath ) ) return;
221  time_t lastModif=fs::last_write_time(dirpath);
222 
223 
224 
225  std::map< std::string, std::string> attr;
227  std::string path = dirpath.c_str();
228  i_sc.addDirectory(path, attr);
229 
230 
231  fs::directory_iterator end_itr; // default construction yields past-the-end
232  for ( fs::directory_iterator itr( dirpath );
233  itr != end_itr;
234  ++itr )
235  {
236 
237  // If is directory & recurse : do recurse
238  if ( fs::is_directory(itr->status()) )
239  {
240  if (recursive)
241  {
242  AddDirectoryRecursorScanner( itr->path().string(), recursive, addKey, i_sc, true);
243  }
244  }
245  else
246  {
247  std::string parent_id;
248  // tTest if directory (and only it) exists or not.
249  bool valid = mSynchronizer->isIndexed(itr->path().string());//true;//=mTimestampHandler->AddDirectory(dirpath, itr->string(), lastModif, time(0),mCurrentDB);
250  if(valid)
251  {
252  std::string path(itr->path().string());
254  boost::algorithm::replace_all( path,INVALID_FILE_SEPARATOR,VALID_FILE_SEPARATOR);
255  i_sc.ReadAttributes(itr->path().string(),attr);
256  // mTreeHandler->GetTopLevelNodeId("FullFileName",itr->string(),parent_id);
257  mTreeHandler->AddBranch(attr);
259  std::stringstream removedOn;
260  removedOn<<time(0);
261  mSynchronizer->InsertIgnoreFile(addKey, path,"0",removedOn.str(),mCurrentDB);
262 
263 
265  if (mProgress.GetStop())
266  {
267  //itr = end_itr;
268  break;
269  }
270  }
271 
272  }
273  }
274  }
275 #endif
276  //=====================================================================
277  void TreeHandlerImageAdder::AddDirectoryRecursor(const std::string &dirpath,
278  bool recursive,
279  const std::string &addKey)
280  {
281  GimmickDebugMessage(4,"Scanning '"<<dirpath<<"'"<<std::endl);
283 
284  if ( !fs::exists( dirpath ) ) return;
285 
287  time_t lastModif=fs::last_write_time(dirpath);
288 
289  fs::directory_iterator end_itr; // default construction yields past-the-end
290  for ( fs::directory_iterator itr( dirpath );
291  itr != end_itr;
292  ++itr )
293  {
294  // If is directory & recurse : do recurse
295  if ( fs::is_directory(itr->status()) )
296  {
297  if (recursive)
298  {
299  AddDirectoryRecursor( itr->path().string(), recursive, addKey);
300  }
301  }
302  else
303  {
304  std::string parent_id;
305  // tTest if directory (and only it) exists or not.
306  bool valid = mSynchronizer->isIndexed(itr->path().string());//true;//=mTimestampHandler->AddDirectory(dirpath, itr->string(), lastModif, time(0),mCurrentDB);
307  if(valid)
308  {
310  if (IsHandledFile(itr->path().string()))
311  {
313  AddFile( itr->path().string() );
314  //mTreeHandler->GetTopLevelNodeId("FullFileName",itr->string(),parent_id);
315  std::stringstream removedOn;
316  removedOn<<time(0);
317  mSynchronizer->InsertIgnoreFile(addKey, itr->path().string(),"0",removedOn.str(),mCurrentDB);
318  }
319 
321  if (mProgress.GetStop())
322  {
323  //itr = end_itr;
324  break;
325  }
326 
327  }
328  }
329  }
330 
331  }
332  //=======================================================================
333 
334 
335  //=======================================================================
336 
337  void TreeHandlerImageAdder::CheckSyncDirectory(const std::string &dirpath,
338  bool recursive,
339  bool repair,
340  bool checkAttributes,
341  std::vector<std::string> &i_ignorefiles,
342  std::vector<std::string> & attsModified,
343  std::vector<std::string> & newfiles)
344  {
345  if ( !fs::exists( dirpath ) ) return;
346  fs::directory_iterator end_itr; // default construction yields past-the-end
347 
348  for ( fs::directory_iterator itr( dirpath ); itr != end_itr; ++itr )
349  {
350  // If is directory & recurse : do recurse
351  if ( fs::is_directory(itr->status()) )
352  {
353  if (recursive)
354  {
355  CheckSyncDirectory( itr->path().string(), recursive, repair, checkAttributes, i_ignorefiles, attsModified, newfiles);
356  }
357  }
358  else
359  {
360  if (IsHandledFile(itr->path().string()))
361  {
362  bool bfound = false;
363  for(std::vector<std::string>::iterator it_new = i_ignorefiles.begin(); it_new < i_ignorefiles.end(); ++it_new)
364  {
365  if((*it_new) == itr->path().string())
366  {
367  bfound = true;
368  //Additional checking of attributes
369  if(checkAttributes)
370  {
371  CheckAttributes(repair,(*it_new),attsModified);
372  }
373  i_ignorefiles.erase(it_new);
374  break;
375  }
376  }
377  if(!bfound && i_ignorefiles.size()>0 )
378  {
379  newfiles.push_back( itr->path().string() );
380  }
381  }
382  }
383  }
384  }
385 
386  //=======================================================================
387 
388  //=======================================================================
389 
390  std::string TreeHandlerImageAdder::Synchronize(bool repair, bool checkAttributes)
391  {
392  std::vector<AddList> fileList;
393  std::vector<std::string> ignoreList;
394  std::vector<std::string> newFiles;
395  std::vector<std::string> attsModified;
396  std::stringstream mess;
397  std::vector<AddList>::iterator iter;
398 
399  //Gets the list of added files
401 
402  std::vector<std::string>::iterator i;
403  //Actions to take if the user doesn't want to repair
404  if(!repair)
405  {
406  //Iterates to see if they are in sync
407  for(iter=fileList.begin();iter!=fileList.end();++iter)
408  {
409  mSynchronizer->GetIgnoredFiles((*iter).key,ignoreList);
410  bool rec=true;
411  if((*iter).recursive=="0"){rec=false;}
412  CheckSyncDirectory((*iter).path,rec,repair,checkAttributes,ignoreList,attsModified,newFiles);
413  }
414 
415  //Add to message the result of new found files
416  mess<<"New Files Found: "<<newFiles.size()<<std::endl;
417  if(newFiles.size()>0)
418  {
419  mess<<"Filenames: "<<std::endl;
420  for(i=newFiles.begin();i!=newFiles.end();++i)
421  {
422  mess<<*i<<std::endl;
423  }
424  }
425 
426  //Add to message the result of missing files
427  mess<<"Missing Files: "<<ignoreList.size()<<std::endl;
428  if(ignoreList.size()>0)
429  {
430  mess<<"Filenames: "<<std::endl;
431  for(i=ignoreList.begin();i!=ignoreList.end();++i)
432  {
433  mess<<*i<<std::endl;
434  }
435  }
436 
437  //In the case that the user wants to check the attributes...
438  if(checkAttributes)
439  {
440  //... add to message the result of files that have been changed.
441  mess<<"Files with different attributes: "<<attsModified.size()<<std::endl;
442  if(attsModified.size()>0)
443  {
444  mess<<"Filenames: "<<std::endl;
445  for(i=attsModified.begin();i!=attsModified.end();++i)
446  {
447  mess<<*i<<std::endl;
448  }
449  }
450  }
451 
452  }
453 
454  //Actions to take if the user wants to repair
455  else
456  {
457  int nf=0;
458  //Iterates to see if they are in sync
459  for(iter=fileList.begin();iter!=fileList.end();++iter)
460  {
461  mSynchronizer->GetIgnoredFiles((*iter).key,ignoreList);
462  bool rec=true;
463  if((*iter).recursive=="0"){rec=false;}
464  CheckSyncDirectory((*iter).path,rec,repair,checkAttributes,ignoreList,attsModified,newFiles);
465 
466  //For the new files, add them
467  for (i=newFiles.begin();i!=newFiles.end();++i)
468  {
469  if (IsHandledFile(*i))
470  {
471  std::stringstream removedOn;
472  removedOn<<time(0);
473  mSynchronizer->InsertIgnoreFile((*iter).key,(*i),"0",removedOn.str(),mCurrentDB);
474  //Gets the number of files added
475  int files=atoi((mSynchronizer->GetAttribute("FILES_ADDED","ADD_OPS","ADD_KEY",(*iter).key,mCurrentDB)).c_str());
476  files=files+1;
477  std::stringstream out;
478  out<<files;
479  //Sets the new number of files
480  mSynchronizer->SetAttribute("FILES_ADDED","ADD_OPS",out.str(),"ADD_KEY",(*iter).key,mCurrentDB);
481  AddFile(*i);
482  }
483  }
484  nf+=(int)newFiles.size();
485  newFiles.clear();
486 
487  }
488  //Reports number of added files
489  mess<<"Files Added: "<<nf<<std::endl;
490 
491  //Removes the necessary files and reports the results
492  if(ignoreList.size()>0)
493  {
494  tree::Node* node;
495  mTreeHandler->LoadChildren(NULL,4);
496  for(i=ignoreList.begin();i!=ignoreList.end();++i)
497  {
498  FindNode(mTreeHandler->GetTree().GetChildrenList()[0],3,"FullFileName",*i,node);
499  RemoveFile(node);
500  mTreeHandler->Remove(node);
501  }
502  }
503  mess<<"Files Removed: "<<ignoreList.size()<<std::endl;
504  //In the case that the user wants to check the attributes...
505  if(checkAttributes)
506  {
507  //... add to message the result of files that were modified.
508  mess<<"Files Modified: "<<attsModified.size()<<std::endl;
509  }
510  }
511  return mess.str();
512 
513  }
514  //=======================================================================
515 
516  void TreeHandlerImageAdder::CheckAttributes(bool repair, std::string& file, std::vector<std::string>& attsModified)
517  {
518  std::map< std::string, std::string> attr;
520  mReader.ReadAttributes(file,attr);
522  tree::LevelDescriptor::AttributeDescriptorListType::const_iterator a;
523  for (a = adl.begin();a!=adl.end();++a)
524  {
525  std::string databaseVal;
526  mTreeHandler->GetAttribute("Image","FullFileName",file,a->GetKey(),databaseVal);
527  std::string fileVal=attr.find(a->GetKey())->second;
528  if ( a->GetFlags()==0 && databaseVal == fileVal)
529  {
530  if(repair)
531  {
532  mTreeHandler->SetAttribute("Image",a->GetKey(),fileVal,"FullFileName", file);
533  }
534  attsModified.push_back(file);
535  }
536 
537  }
538 
539  }
540 
541  //=======================================================================
542 
543 
544  void TreeHandlerImageAdder::FindNode(tree::Node* parent, int level, const std::string& searchParam, const std::string& searchVal, tree::Node*& node)
545  {
546  if(level>1)
547  {
548  std::vector<tree::Node*>::iterator iter;
549  for(iter=parent->GetChildrenList().begin();iter!=parent->GetChildrenList().end();++iter)
550  {
551  FindNode(*iter,level-1,searchParam,searchVal,node);
552  }
553  }
554  else
555  {
556  if(parent->GetAttribute(searchParam).compare(searchVal)==0)
557  {
558  node=parent;
559  }
560 
561  }
562  }
563 
564  void TreeHandlerImageAdder::SaveAs(const std::vector<std::string>& filenames, std::vector<vtkImageData *> i_images)
565  {
566  std::vector<std::string>::const_iterator it_file = filenames.begin();
567  std::vector<vtkImageData *>::iterator it_image = i_images.begin();
568  /* mWriter.CanWrite(".jpeg");
569  for(; it_file != filenames.end(); ++it_file, ++it_image)
570  mWriter.WriteImage(it_file->c_str(), (vtkImageData &)it_image);*/
571  }
572 
573  //=======================================================================
574  void TreeHandlerImageAdder::FindNodePartial(tree::Node* parent, int level, const std::string& searchParam, const std::string& searchVal, tree::Node*& node)
575  {
576  if(level>1)
577  {
578  std::vector<tree::Node*>::iterator iter;
579  for(iter=parent->GetChildrenList().begin();iter!=parent->GetChildrenList().end() && node==0 ;++iter)
580  {
581  FindNodePartial(*iter,level-1,searchParam,searchVal,node);
582  }
583  }
584  else
585  {
586  if(parent->GetAttribute(searchParam).find(searchVal)<9000)
587  {
588  node=parent;
589  return;
590  }
591 
592  }
593  }
594 
595  //=======================================================================
596 
597  void TreeHandlerImageAdder::CopyFiles(const std::vector<std::string>& filenames, const std::string directory )
598  {
599  std::vector<std::string>::const_iterator i;
600  if(!boost::filesystem::exists(directory))
601  {
602  boost::filesystem::create_directory(boost::filesystem::path(directory));
603  mSynchronizer->InsertAddOp(directory,"0","0",mCurrentDB);
604  }
605  std::string addKey=mSynchronizer->GetAttribute("ADD_KEY","ADD_OPS","PATH",directory,mCurrentDB);
606  size_t last;
607  std::vector<std::string> newNames;
608  for(i=filenames.begin();i!=filenames.end();++i)
609  {
610  std::string dir=directory.c_str();
611  if(boost::filesystem::exists(*i) && (*i).find(dir)==std::string::npos)
612  {
613  std::string path=*i;
614  last=(*i).find_last_of('/');
615  std::string f="\\"+(*i).substr(last+1);
616 
617  int p=1;
618  std::stringstream out;
619  out<<directory<<f;
620  while(boost::filesystem::exists(out.str()))
621  {
622  out.str("");
623  out<<directory<<f.substr(0,f.size()-4)<<"("<<p<<")"<<f.substr(f.size()-4);
624  p++;
625  }
626  std::string result=out.str();
627  boost::filesystem::copy_file((*i),result);
628 
629  //To update image database
630  mTreeHandler->SetAttribute("Image","FullFileName",result,"FullFileName", (*i));
631 
632  //To update maintenance database
633  //1.Add the new path and increase number of children on new operation.
634  std::stringstream removedOn;
635  removedOn<<time(0);
636  //Inserts the file
637  mSynchronizer->InsertIgnoreFile(addKey, result,"0",removedOn.str(),mCurrentDB);
638  //Gets the number of files added
639  int files=atoi((mSynchronizer->GetAttribute("FILES_ADDED","ADD_OPS","ADD_KEY",addKey,mCurrentDB)).c_str());
640  files=files+1;
641  std::stringstream fil;
642  fil<<files;
643  //Sets the new number of files
644  mSynchronizer->SetAttribute("FILES_ADDED","ADD_OPS",fil.str(),"ADD_KEY",addKey,mCurrentDB);
645  fil.str("");
646 
647  //2.Set the old path as removed and decrease number of children on old operation.
648  //Gets the old add key
649  std::string oldAddKey=mSynchronizer->GetAttribute("ADD_KEY","IGNORED_FILES","PATH",path,mCurrentDB);
650  //Sets the file as removed
651  mSynchronizer->SetAttribute("REMOVE","IGNORED_FILES","1","PATH = '"+path+"' AND ADD_KEY",oldAddKey,mCurrentDB);
652  //Gets the number of files added
653  files=atoi((mSynchronizer->GetAttribute("FILES_ADDED","ADD_OPS","ADD_KEY",oldAddKey,mCurrentDB)).c_str());
654  files=files-1;
655  fil<<files;
656  //Sets the new number of files
657  mSynchronizer->SetAttribute("FILES_ADDED","ADD_OPS",fil.str(),"ADD_KEY",oldAddKey,mCurrentDB);
658 
659  }
660 
661  }
662  }
663 
664  //=======================================================================
665 
666  void TreeHandlerImageAdder::DeleteDriveFromMainDB(const std::string& drive)
667  {
668  //Delete from local database and others
669  tree::Node* node=0;
670  mTreeHandler->LoadChildren(NULL,4);
671  FindNodePartial(mTreeHandler->GetTree().GetChildrenList()[0],3,"FullFileName",drive,node);
672  while(node!=0)
673  {
674  mTreeHandler->Remove(node);
675  node=0;
676  mTreeHandler->LoadChildren(NULL,4);
677  FindNodePartial(mTreeHandler->GetTree().GetChildrenList()[0],3,"FullFileName",drive,node);
678  }
679  }
680 
681  //=======================================================================
682 
683  void TreeHandlerImageAdder::DeleteDriveFromOtherDB(const std::string& drive)
684  {
685  //Delete from maintenance
686  mSynchronizer->RemoveEntries("ADD_OPS", "PATH", "LIKE", drive+"%");
687  mSynchronizer->RemoveEntries("IGNORED_FILES", "PATH", "LIKE", drive+"%");
688  }
689 
690  //=======================================================================
691  void TreeHandlerImageAdder::EditField(tree::Node* node, const std::string& name, const std::string& key, const std::string& val)
692  {
693  node->SetAttribute(key,val);
694  mTreeHandler->SetAttribute(node,key,val);
695  }
696 
697  //=======================================================================
698  void TreeHandlerImageAdder::GetAttributes(const std::vector<std::string>& params,
699  const std::string& filename,
700  std::vector<std::string>& results)
701  {
702  std::vector<std::string>::const_iterator i;
703  std::string result;
704  for(i=params.begin();i!=params.end();i++)
705  {
706  mTreeHandler->GetAttribute("Image","FullFileName",filename,*i,result);
707  results.push_back(result);
708  }
709  }
710 
711  //=======================================================================
712  const std::string TreeHandlerImageAdder::isAttributeExist(const std::string i_attr)
713  {
714  return mTreeHandler->GetTree().isAttributeExist(i_attr);
715  }
716 
717 }