MythTV  master
remotefile.cpp
Go to the documentation of this file.
1 #include <iostream>
2 using namespace std;
3 
4 #include <QUrl>
5 #include <QFile>
6 #include <QFileInfo>
7 
8 // POSIX C headers
9 #include <unistd.h>
10 #include <fcntl.h>
11 
12 #include "mythconfig.h"
13 
14 #ifndef O_STREAMING
15 #define O_STREAMING 0
16 #endif
17 
18 #ifndef O_LARGEFILE
19 #define O_LARGEFILE 0
20 #endif
21 
22 #ifndef O_BINARY
23 #define O_BINARY 0
24 #endif
25 
26 #include "mythdb.h"
27 #include "remotefile.h"
28 #include "mythcorecontext.h"
29 #include "mythsocket.h"
30 #include "compat.h"
31 #include "mythtimer.h"
32 #include "mythdate.h"
33 #include "mythmiscutil.h"
34 #include "threadedfilewriter.h"
35 #include "storagegroup.h"
36 
37 #define MAX_FILE_CHECK 500 // in ms
38 
39 static bool RemoteSendReceiveStringList(const QString &host, QStringList &strlist)
40 {
41  bool ok = false;
42 
44  {
45  // since the master backend cannot connect back around to
46  // itself, and the libraries do not have access to the list
47  // of connected slave backends to query an existing connection
48  // start up a new temporary connection directly to the slave
49  // backend to query the file list
50  QString ann = QString("ANN Playback %1 0")
51  .arg(gCoreContext->GetHostName());
52  QString addr = gCoreContext->GetBackendServerIP(host);
53  int port = gCoreContext->GetBackendServerPort(host);
54  bool mismatch = false;
55 
57  addr, port, ann, &mismatch);
58  if (sock)
59  {
60  ok = sock->SendReceiveStringList(strlist);
61  sock->DecrRef();
62  }
63  else
64  strlist.clear();
65  }
66  else
67  ok = gCoreContext->SendReceiveStringList(strlist);
68 
69  return ok;
70 }
71 
72 RemoteFile::RemoteFile(const QString &url, bool write, bool usereadahead,
73  int timeout_ms,
74  const QStringList *possibleAuxiliaryFiles) :
75  m_path(url),
76  m_usereadahead(usereadahead), m_timeout_ms(timeout_ms),
77  m_writemode(write)
78 {
79  if (m_writemode)
80  {
81  m_usereadahead = false;
82  m_timeout_ms = -1;
83  }
84  else if (possibleAuxiliaryFiles)
85  m_possibleauxfiles = *possibleAuxiliaryFiles;
86 
87  if (!m_path.isEmpty())
88  Open();
89 
90  LOG(VB_FILE, LOG_DEBUG, QString("RemoteFile(%1)").arg(m_path));
91 }
92 
94 {
95  Close();
96  if (m_controlSock)
97  {
99  m_controlSock = nullptr;
100  }
101  if (m_sock)
102  {
103  m_sock->DecrRef();
104  m_sock = nullptr;
105  }
106 }
107 
108 bool RemoteFile::isLocal(const QString &lpath)
109 {
110  bool is_local = !lpath.isEmpty() &&
111  !lpath.startsWith("myth:") &&
112  (lpath.startsWith("/") || QFile::exists(lpath));
113  return is_local;
114 }
115 
116 bool RemoteFile::isLocal(void) const
117 {
118  return isLocal(m_path);
119 }
120 
122 {
123  QUrl qurl(m_path);
124  QString dir;
125 
126  QString host = qurl.host();
127  int port = qurl.port();
128 
129  dir = qurl.path();
130 
131  if (qurl.hasQuery())
132  dir += "?" + QUrl::fromPercentEncoding(
133  qurl.query(QUrl::FullyEncoded).toLocal8Bit());
134 
135  if (qurl.hasFragment())
136  dir += "#" + qurl.fragment();
137 
138  QString sgroup = qurl.userName();
139 
140  MythSocket *lsock = new MythSocket();
141  QString stype = (control) ? "control socket" : "file data socket";
142 
143  QString loc = QString("RemoteFile::openSocket(%1): ").arg(stype);
144 
145  if (port <= 0)
146  {
147  port = gCoreContext->GetBackendServerPort(host);
148  }
149 
150  if (!lsock->ConnectToHost(host, port))
151  {
152  LOG(VB_GENERAL, LOG_ERR, loc +
153  QString("Could not connect to server %1:%2") .arg(host).arg(port));
154  lsock->DecrRef();
155  return nullptr;
156  }
157 
158  QString hostname = GetMythDB()->GetHostName();
159 
160  QStringList strlist;
161 
162 #ifndef IGNORE_PROTO_VER_MISMATCH
163  if (!gCoreContext->CheckProtoVersion(lsock, 5000))
164  {
165  LOG(VB_GENERAL, LOG_ERR, loc +
166  QString("Failed validation to server %1:%2").arg(host).arg(port));
167  lsock->DecrRef();
168  return nullptr;
169  }
170 #endif
171 
172  if (control)
173  {
174  strlist.append(QString("ANN Playback %1 %2").arg(hostname).arg(false));
175  if (!lsock->SendReceiveStringList(strlist))
176  {
177  LOG(VB_GENERAL, LOG_ERR, loc +
178  QString("Could not read string list from server %1:%2")
179  .arg(host).arg(port));
180  lsock->DecrRef();
181  return nullptr;
182  }
183  }
184  else
185  {
186  strlist.push_back(QString("ANN FileTransfer %1 %2 %3 %4")
187  .arg(hostname).arg(m_writemode)
188  .arg(m_usereadahead).arg(m_timeout_ms));
189  strlist << QString("%1").arg(dir);
190  strlist << sgroup;
191 
192  QStringList::const_iterator it = m_possibleauxfiles.begin();
193  for (; it != m_possibleauxfiles.end(); ++it)
194  strlist << *it;
195 
196  if (!lsock->SendReceiveStringList(strlist))
197  {
198  LOG(VB_GENERAL, LOG_ERR, loc +
199  QString("Did not get proper response from %1:%2")
200  .arg(host).arg(port));
201  strlist.clear();
202  strlist.push_back("ERROR");
203  strlist.push_back("invalid response");
204  }
205 
206  if (strlist.size() >= 3)
207  {
208  it = strlist.begin(); ++it;
209  m_recordernum = (*it).toInt(); ++it;
210  m_filesize = (*(it)).toLongLong(); ++it;
211  for (; it != strlist.end(); ++it)
212  m_auxfiles << *it;
213  }
214  else if (!strlist.isEmpty() && strlist.size() < 3 &&
215  strlist[0] != "ERROR")
216  {
217  LOG(VB_GENERAL, LOG_ERR, loc +
218  QString("Did not get proper response from %1:%2")
219  .arg(host).arg(port));
220  strlist.clear();
221  strlist.push_back("ERROR");
222  strlist.push_back("invalid response");
223  }
224  }
225 
226  if (strlist.isEmpty() || strlist[0] == "ERROR")
227  {
228  lsock->DecrRef();
229  lsock = nullptr;
230  if (strlist.isEmpty())
231  {
232  LOG(VB_GENERAL, LOG_ERR, loc + "Failed to open socket, timeout");
233  }
234  else
235  {
236  LOG(VB_GENERAL, LOG_ERR, loc + "Failed to open socket" +
237  ((strlist.size() >= 2) ?
238  QString(", error was %1").arg(strlist[1]) :
239  QString(", remote error")));
240  }
241  }
242 
243  return lsock;
244 }
245 
246 bool RemoteFile::isOpen() const
247 {
248  if (isLocal())
249  {
250  return m_writemode ? (m_fileWriter != nullptr) : (m_localFile != -1);
251  }
252  return m_sock && m_controlSock;
253 }
254 
256 {
257  if (isOpen())
258  return true;
259 
260  QMutexLocker locker(&m_lock);
261  return OpenInternal();
262 }
263 
269 {
270  if (isLocal())
271  {
272  if (m_writemode)
273  {
274  // make sure the directories are created if necessary
275  QFileInfo fi(m_path);
276  QDir dir(fi.path());
277  if (!dir.exists())
278  {
279  LOG(VB_FILE, LOG_WARNING, QString("RemoteFile::Open(%1) creating directories")
280  .arg(m_path));
281 
282  if (!dir.mkpath(fi.path()))
283  {
284  LOG(VB_GENERAL, LOG_ERR, QString("RemoteFile::Open(%1) failed to create the directories")
285  .arg(m_path));
286  return false;
287  }
288  }
289 
291  O_WRONLY|O_TRUNC|O_CREAT|O_LARGEFILE,
292  0644);
293 
294  if (!m_fileWriter->Open())
295  {
296  delete m_fileWriter;
297  m_fileWriter = nullptr;
298  LOG(VB_FILE, LOG_ERR, QString("RemoteFile::Open(%1) write mode error")
299  .arg(m_path));
300  return false;
301  }
302  SetBlocking();
303  return true;
304  }
305 
306  // local mode, read only
307  if (!Exists(m_path))
308  {
309  LOG(VB_FILE, LOG_ERR,
310  QString("RemoteFile::Open(%1) Error: Does not exist").arg(m_path));
311  return false;
312  }
313 
314  m_localFile = ::open(m_path.toLocal8Bit().constData(), O_RDONLY);
315  if (m_localFile == -1)
316  {
317  LOG(VB_FILE, LOG_ERR, QString("RemoteFile::Open(%1) Error: %2")
318  .arg(m_path).arg(strerror(errno)));
319  return false;
320  }
321  return true;
322  }
323  m_controlSock = openSocket(true);
324  if (!m_controlSock)
325  return false;
326 
327  m_sock = openSocket(false);
328  if (!m_sock)
329  {
330  // Close the sockets if we received an error so that isOpen() will
331  // return false if the caller tries to use the RemoteFile.
332  Close(true);
333  return false;
334  }
335  m_canresume = true;
336 
337  return true;
338 }
339 
340 bool RemoteFile::ReOpen(const QString& newFilename)
341 {
342  if (isLocal())
343  {
344  if (isOpen())
345  {
346  Close();
347  }
348  m_path = newFilename;
349  return Open();
350  }
351 
352  QMutexLocker locker(&m_lock);
353 
354  if (!CheckConnection(false))
355  {
356  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::ReOpen(): Couldn't connect");
357  return false;
358  }
359 
360  QStringList strlist( m_query.arg(m_recordernum) );
361  strlist << "REOPEN";
362  strlist << newFilename;
363 
365 
366  m_lock.unlock();
367 
368  bool retval = false;
369  if (!strlist.isEmpty())
370  retval = (strlist[0].toInt() != 0);
371 
372  return retval;
373 }
374 
375 void RemoteFile::Close(bool haslock)
376 {
377  if (isLocal())
378  {
380  m_localFile = -1;
381  delete m_fileWriter;
382  m_fileWriter = nullptr;
383  return;
384  }
385  if (!m_controlSock)
386  return;
387 
388  QStringList strlist( m_query.arg(m_recordernum) );
389  strlist << "DONE";
390 
391  if (!haslock)
392  {
393  m_lock.lock();
394  }
396  strlist, 0, MythSocket::kShortTimeout))
397  {
398  LOG(VB_GENERAL, LOG_ERR, "Remote file timeout.");
399  }
400 
401  if (m_sock)
402  {
403  m_sock->DecrRef();
404  m_sock = nullptr;
405  }
406  if (m_controlSock)
407  {
409  m_controlSock = nullptr;
410  }
411 
412  if (!haslock)
413  {
414  m_lock.unlock();
415  }
416 }
417 
418 bool RemoteFile::DeleteFile(const QString &url)
419 {
420  if (isLocal(url))
421  {
422  QFile file(url);
423  return file.remove();
424  }
425 
426  bool result = false;
427  QUrl qurl(url);
428  QString filename = qurl.path();
429  QString sgroup = qurl.userName();
430 
431  if (!qurl.fragment().isEmpty() || url.endsWith("#"))
432  filename = filename + "#" + qurl.fragment();
433 
434  if (filename.startsWith("/"))
435  filename = filename.right(filename.length()-1);
436 
437  if (filename.isEmpty() || sgroup.isEmpty())
438  return false;
439 
440  QStringList strlist("DELETE_FILE");
441  strlist << filename;
442  strlist << sgroup;
443 
445 
446  if (!strlist.isEmpty() && strlist[0] == "1")
447  result = true;
448 
449  return result;
450 }
451 
452 bool RemoteFile::Exists(const QString &url)
453 {
454  if (url.isEmpty())
455  return false;
456 
457  struct stat fileinfo;
458  return Exists(url, &fileinfo);
459 }
460 
461 bool RemoteFile::Exists(const QString &url, struct stat *fileinfo)
462 {
463  if (url.isEmpty())
464  return false;
465 
466  QUrl qurl(url);
467  QString filename = qurl.path();
468  QString sgroup = qurl.userName();
469  QString host = qurl.host();
470 
471  if (isLocal(url) || gCoreContext->IsThisBackend(host))
472  {
473  LOG(VB_FILE, LOG_INFO,
474  QString("RemoteFile::Exists(): looking for local file: %1").arg(url));
475 
476  bool fileExists = false;
477  QString fullFilePath = "";
478 
479  if (url.startsWith("myth:"))
480  {
481  StorageGroup sGroup(sgroup, gCoreContext->GetHostName());
482  fullFilePath = sGroup.FindFile(filename);
483  if (!fullFilePath.isEmpty())
484  fileExists = true;
485  }
486  else
487  {
488  QFileInfo info(url);
489  fileExists = info.exists() /*&& info.isFile()*/;
490  fullFilePath = url;
491  }
492 
493  if (fileExists)
494  {
495  if (stat(fullFilePath.toLocal8Bit().constData(), fileinfo) == -1)
496  {
497  LOG(VB_FILE, LOG_ERR,
498  QString("RemoteFile::Exists(): failed to stat file: %1").arg(fullFilePath) + ENO);
499  }
500  }
501 
502  return fileExists;
503  }
504 
505  LOG(VB_FILE, LOG_INFO,
506  QString("RemoteFile::Exists(): looking for remote file: %1").arg(url));
507 
508  if (!qurl.fragment().isEmpty() || url.endsWith("#"))
509  filename = filename + "#" + qurl.fragment();
510 
511  if (filename.startsWith("/"))
512  filename = filename.right(filename.length()-1);
513 
514  if (filename.isEmpty())
515  return false;
516 
517  QStringList strlist("QUERY_FILE_EXISTS");
518  strlist << filename;
519  if (!sgroup.isEmpty())
520  strlist << sgroup;
521 
522  bool result = false;
523  if (RemoteSendReceiveStringList(host, strlist) && strlist[0] == "1")
524  {
525  if ((strlist.size() >= 15) && fileinfo)
526  {
527  fileinfo->st_dev = strlist[2].toLongLong();
528  fileinfo->st_ino = strlist[3].toLongLong();
529  fileinfo->st_mode = strlist[4].toLongLong();
530  fileinfo->st_nlink = strlist[5].toLongLong();
531  fileinfo->st_uid = strlist[6].toLongLong();
532  fileinfo->st_gid = strlist[7].toLongLong();
533  fileinfo->st_rdev = strlist[8].toLongLong();
534  fileinfo->st_size = strlist[9].toLongLong();
535 #ifndef _WIN32
536  fileinfo->st_blksize = strlist[10].toLongLong();
537  fileinfo->st_blocks = strlist[11].toLongLong();
538 #endif
539  fileinfo->st_atime = strlist[12].toLongLong();
540  fileinfo->st_mtime = strlist[13].toLongLong();
541  fileinfo->st_ctime = strlist[14].toLongLong();
542  result = true;
543  }
544  else if (!fileinfo)
545  {
546  result = true;
547  }
548  }
549 
550  return result;
551 }
552 
553 QString RemoteFile::GetFileHash(const QString &url)
554 {
555  if (isLocal(url))
556  {
557  return FileHash(url);
558  }
559  QString result;
560  QUrl qurl(url);
561  QString filename = qurl.path();
562  QString hostname = qurl.host();
563  QString sgroup = qurl.userName();
564 
565  if (!qurl.fragment().isEmpty() || url.endsWith("#"))
566  filename = filename + "#" + qurl.fragment();
567 
568  if (filename.startsWith("/"))
569  filename = filename.right(filename.length()-1);
570 
571  if (filename.isEmpty() || sgroup.isEmpty())
572  return QString();
573 
574  QStringList strlist("QUERY_FILE_HASH");
575  strlist << filename;
576  strlist << sgroup;
577  strlist << hostname;
578 
580  {
581  result = strlist[0];
582  }
583 
584  return result;
585 }
586 
587 bool RemoteFile::CopyFile (const QString& src, const QString& dst,
588  bool overwrite, bool verify)
589 {
590  LOG(VB_FILE, LOG_INFO,
591  QString("RemoteFile::CopyFile: Copying file from '%1' to '%2'").arg(src).arg(dst));
592 
593  // sanity check
594  if (src == dst)
595  {
596  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::CopyFile: Cannot copy a file to itself");
597  return false;
598  }
599 
600  RemoteFile srcFile(src, false);
601  if (!srcFile.isOpen())
602  {
603  LOG(VB_GENERAL, LOG_ERR,
604  QString("RemoteFile::CopyFile: Failed to open file (%1) for reading.").arg(src));
605  return false;
606  }
607 
608  const int readSize = 2 * 1024 * 1024;
609  char *buf = new char[readSize];
610  if (!buf)
611  {
612  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::CopyFile: ERROR, unable to allocate copy buffer");
613  return false;
614  }
615 
616  if (overwrite)
617  {
618  DeleteFile(dst);
619  }
620  else if (Exists(dst))
621  {
622  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::CopyFile: File already exists");
623  delete[] buf;
624  return false;
625  }
626 
627  RemoteFile dstFile(dst, true);
628  if (!dstFile.isOpen())
629  {
630  LOG(VB_GENERAL, LOG_ERR,
631  QString("RemoteFile::CopyFile: Failed to open file (%1) for writing.").arg(dst));
632  srcFile.Close();
633  delete[] buf;
634  return false;
635  }
636 
637  dstFile.SetBlocking(true);
638 
639  bool success = true;
640  int srcLen;
641 
642  while ((srcLen = srcFile.Read(buf, readSize)) > 0)
643  {
644  int dstLen = dstFile.Write(buf, srcLen);
645 
646  if (dstLen == -1 || srcLen != dstLen)
647  {
648  LOG(VB_GENERAL, LOG_ERR,
649  "RemoteFile::CopyFile: Error while trying to write to destination file.");
650  success = false;
651  }
652  }
653 
654  srcFile.Close();
655  dstFile.Close();
656  delete[] buf;
657 
658  if (success && verify)
659  {
660  // Check written file is correct size
661  struct stat fileinfo;
662  long long dstSize = Exists(dst, &fileinfo) ? fileinfo.st_size : -1;
663  long long srcSize = srcFile.GetFileSize();
664  if (dstSize != srcSize)
665  {
666  LOG(VB_GENERAL, LOG_ERR,
667  QString("RemoteFile::CopyFile: Copied file is wrong size (%1 rather than %2)")
668  .arg(dstSize).arg(srcSize));
669  success = false;
670  DeleteFile(dst);
671  }
672  }
673 
674  return success;
675 }
676 
677 bool RemoteFile::MoveFile (const QString& src, const QString& dst, bool overwrite)
678 {
679  LOG(VB_FILE, LOG_INFO,
680  QString("RemoteFile::MoveFile: Moving file from '%1' to '%2'").arg(src).arg(dst));
681 
682  // sanity check
683  if (src == dst)
684  {
685  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::MoveFile: Cannot move a file to itself");
686  return false;
687  }
688 
689  if (isLocal(src) != isLocal(dst))
690  {
691  // Moving between local & remote requires a copy & delete
692  bool ok = CopyFile(src, dst, overwrite, true);
693  if (ok)
694  {
695  if (!DeleteFile(src))
696  LOG(VB_FILE, LOG_ERR,
697  "RemoteFile::MoveFile: Failed to delete file after successful copy");
698  }
699  return ok;
700  }
701 
702  if (overwrite)
703  {
704  DeleteFile(dst);
705  }
706  else if (Exists(dst))
707  {
708  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::MoveFile: File already exists");
709  return false;
710  }
711 
712  if (isLocal(src))
713  {
714  // Moving local -> local
715  QFileInfo fi(dst);
716  if (QDir().mkpath(fi.path()) && QFile::rename(src, dst))
717  return true;
718 
719  LOG(VB_FILE, LOG_ERR, "RemoteFile::MoveFile: Rename failed");
720  return false;
721  }
722 
723  // Moving remote -> remote
724  QUrl srcUrl(src);
725  QUrl dstUrl(dst);
726 
727  if (srcUrl.userName() != dstUrl.userName())
728  {
729  LOG(VB_FILE, LOG_ERR, "RemoteFile::MoveFile: Cannot change a file's Storage Group");
730  return false;
731  }
732 
733  QStringList strlist("MOVE_FILE");
734  strlist << srcUrl.userName() << srcUrl.path() << dstUrl.path();
735 
737 
738  if (!strlist.isEmpty() && strlist[0] == "1")
739  return true;
740 
741  LOG(VB_FILE, LOG_ERR, QString("RemoteFile::MoveFile: MOVE_FILE failed with: %1")
742  .arg(strlist.join(",")));
743  return false;
744 }
745 
747 {
748  if (isLocal())
749  return;
750  QMutexLocker locker(&m_lock);
751  if (!m_sock)
752  {
753  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::Reset(): Called with no socket");
754  return;
755  }
756  m_sock->Reset();
757 }
758 
759 long long RemoteFile::Seek(long long pos, int whence, long long curpos)
760 {
761  QMutexLocker locker(&m_lock);
762 
763  return SeekInternal(pos, whence, curpos);
764 }
765 
766 long long RemoteFile::SeekInternal(long long pos, int whence, long long curpos)
767 {
768  if (isLocal())
769  {
770  if (!isOpen())
771  {
772  LOG(VB_FILE, LOG_ERR, "RemoteFile::Seek(): Called with no file opened");
773  return -1;
774  }
775  if (m_writemode)
776  return m_fileWriter->Seek(pos, whence);
777 
778  long long offset = 0LL;
779  if (whence == SEEK_SET)
780  {
781  QFileInfo info(m_path);
782  offset = min(pos, info.size());
783  }
784  else if (whence == SEEK_END)
785  {
786  QFileInfo info(m_path);
787  offset = info.size() + pos;
788  }
789  else if (whence == SEEK_CUR)
790  {
791  offset = ((curpos > 0) ? curpos : ::lseek64(m_localFile, 0, SEEK_CUR)) + pos;
792  }
793  else
794  return -1;
795 
796  off64_t localpos = ::lseek64(m_localFile, pos, whence);
797  if (localpos != pos)
798  {
799  LOG(VB_FILE, LOG_ERR,
800  QString("RemoteFile::Seek(): Couldn't seek to offset %1")
801  .arg(offset));
802  return -1;
803  }
804  return localpos;
805  }
806 
807  if (!CheckConnection(false))
808  {
809  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::Seek(): Couldn't connect");
810  return -1;
811  }
812 
813  QStringList strlist( m_query.arg(m_recordernum) );
814  strlist << "SEEK";
815  strlist << QString::number(pos);
816  strlist << QString::number(whence);
817  if (curpos > 0)
818  strlist << QString::number(curpos);
819  else
820  strlist << QString::number(m_readposition);
821 
822  bool ok = m_controlSock->SendReceiveStringList(strlist);
823 
824  if (ok && !strlist.isEmpty())
825  {
826  m_lastposition = m_readposition = strlist[0].toLongLong();
827  m_sock->Reset();
828  return strlist[0].toLongLong();
829  }
830  m_lastposition = 0LL;
831  return -1;
832 }
833 
834 int RemoteFile::Write(const void *data, int size)
835 {
836  int recv = 0;
837  int sent = 0;
838  unsigned zerocnt = 0;
839  bool error = false;
840  bool response = false;
841 
842  if (!m_writemode)
843  {
844  LOG(VB_NETWORK, LOG_ERR,
845  "RemoteFile::Write(): Called when not in write mode");
846  return -1;
847  }
848  if (isLocal())
849  {
850  if (!isOpen())
851  {
852  LOG(VB_FILE, LOG_ERR,
853  "RemoteFile::Write(): File not opened");
854  return -1;
855  }
856  return m_fileWriter->Write(data, size);
857  }
858 
859  QMutexLocker locker(&m_lock);
860 
861  if (!CheckConnection())
862  {
863  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::Write(): Couldn't connect");
864  return -1;
865  }
866 
867  QStringList strlist( m_query.arg(m_recordernum) );
868  strlist << "WRITE_BLOCK";
869  strlist << QString::number(size);
870  bool ok = m_controlSock->WriteStringList(strlist);
871  if (!ok)
872  {
873  LOG(VB_NETWORK, LOG_ERR,
874  "RemoteFile::Write(): Block notification failed");
875  return -1;
876  }
877 
878  recv = size;
879  while (sent < recv && !error && zerocnt++ < 50)
880  {
881  int ret = m_sock->Write((char*)data + sent, recv - sent);
882  if (ret > 0)
883  {
884  sent += ret;
885  }
886  else
887  {
888  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::Write(): socket error");
889  error = true;
890  break;
891  }
892 
895  !strlist.isEmpty())
896  {
897  recv = strlist[0].toInt(); // -1 on backend error
898  response = true;
899  }
900  }
901 
902  if (!error && !response)
903  {
905  !strlist.isEmpty())
906  {
907  recv = strlist[0].toInt(); // -1 on backend error
908  }
909  else
910  {
911  LOG(VB_GENERAL, LOG_ERR,
912  "RemoteFile::Write(): No response from control socket.");
913  recv = -1;
914  }
915  }
916 
917  LOG(VB_NETWORK, LOG_DEBUG,
918  QString("RemoteFile::Write(): reqd=%1, sent=%2, rept=%3, error=%4")
919  .arg(size).arg(sent).arg(recv).arg(error));
920 
921  if (recv < 0)
922  return recv;
923 
924  if (error || recv != sent)
925  {
926  sent = -1;
927  }
928  else
929  {
930  m_lastposition += sent;
931  }
932 
933  return sent;
934 }
935 
936 int RemoteFile::Read(void *data, int size)
937 {
938  int recv = 0;
939  int sent = 0;
940  bool error = false;
941  bool response = false;
942 
943  QMutexLocker locker(&m_lock);
944 
945  if (isLocal())
946  {
947  if (m_writemode)
948  {
949  LOG(VB_FILE, LOG_ERR, "RemoteFile:Read() called in writing mode");
950  return -1;
951  }
952  if (isOpen())
953  {
954  return ::read(m_localFile, data, size);
955  }
956  LOG(VB_FILE, LOG_ERR, "RemoteFile:Read() called when local file not opened");
957  return -1;
958  }
959 
960  if (!CheckConnection())
961  {
962  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::Read(): Couldn't connect");
963  return -1;
964  }
965 
966  if (m_sock->IsDataAvailable())
967  {
968  LOG(VB_NETWORK, LOG_ERR,
969  "RemoteFile::Read(): Read socket not empty to start!");
970  m_sock->Reset();
971  }
972 
973  while (m_controlSock->IsDataAvailable())
974  {
975  LOG(VB_NETWORK, LOG_WARNING,
976  "RemoteFile::Read(): Control socket not empty to start!");
977  m_controlSock->Reset();
978  }
979 
980  QStringList strlist( m_query.arg(m_recordernum) );
981  strlist << "REQUEST_BLOCK";
982  strlist << QString::number(size);
983  bool ok = m_controlSock->WriteStringList(strlist);
984  if (!ok)
985  {
986  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::Read(): Block request failed");
987  return -1;
988  }
989 
990  sent = size;
991 
992  int waitms = 30;
993  MythTimer mtimer;
994  mtimer.start();
995 
996  while (recv < sent && !error && mtimer.elapsed() < 10000)
997  {
998  int ret = m_sock->Read(((char *)data) + recv, sent - recv, waitms);
999 
1000  if (ret > 0)
1001  recv += ret;
1002  else if (ret < 0)
1003  error = true;
1004 
1005  waitms += (waitms < 200) ? 20 : 0;
1006 
1007  if (m_controlSock->IsDataAvailable() &&
1009  !strlist.isEmpty())
1010  {
1011  sent = strlist[0].toInt(); // -1 on backend error
1012  response = true;
1013  if (ret < sent)
1014  {
1015  // We have received less than what the server sent, retry immediately
1016  ret = m_sock->Read(((char *)data) + recv, sent - recv, waitms);
1017  if (ret > 0)
1018  recv += ret;
1019  else if (ret < 0)
1020  error = true;
1021  }
1022  }
1023  }
1024 
1025  if (!error && !response)
1026  {
1027  // Wait up to 1.5s for the backend to send the size
1028  // MythSocket::ReadString will drop the connection
1029  if (m_controlSock->ReadStringList(strlist, 1500) &&
1030  !strlist.isEmpty())
1031  {
1032  sent = strlist[0].toInt(); // -1 on backend error
1033  }
1034  else
1035  {
1036  LOG(VB_GENERAL, LOG_ERR,
1037  "RemoteFile::Read(): No response from control socket.");
1038  // If no data was received from control socket, and we got what we asked for
1039  // assume everything is okay
1040  if (recv == size)
1041  {
1042  sent = recv;
1043  }
1044  else
1045  {
1046  sent = -1;
1047  }
1048  // The TCP socket is dropped if there's a timeout, so we reconnect
1049  if (!Resume())
1050  {
1051  sent = -1;
1052  }
1053  }
1054  }
1055 
1056  LOG(VB_NETWORK, LOG_DEBUG,
1057  QString("Read(): reqd=%1, rcvd=%2, rept=%3, error=%4")
1058  .arg(size).arg(recv).arg(sent).arg(error));
1059 
1060  if (sent < 0)
1061  return sent;
1062 
1063  if (error || sent != recv)
1064  {
1065  LOG(VB_GENERAL, LOG_WARNING,
1066  QString("RemoteFile::Read(): sent %1 != recv %2")
1067  .arg(sent).arg(recv));
1068  recv = -1;
1069 
1070  // The TCP socket is dropped if there's a timeout, so we reconnect
1071  if (!Resume())
1072  {
1073  LOG(VB_GENERAL, LOG_WARNING, "RemoteFile::Read(): Resume failed.");
1074  }
1075  else
1076  LOG(VB_GENERAL, LOG_NOTICE, "RemoteFile::Read(): Resume success.");
1077  }
1078  else
1079  {
1080  m_lastposition += recv;
1081  }
1082 
1083  return recv;
1084 }
1085 
1091 long long RemoteFile::GetFileSize(void) const
1092 {
1093  if (isLocal())
1094  {
1095  if (isOpen() && m_writemode)
1096  {
1097  m_fileWriter->Flush();
1098  }
1099  if (Exists(m_path))
1100  {
1101  QFileInfo info(m_path);
1102  return info.size();
1103  }
1104  return -1;
1105  }
1106 
1107  QMutexLocker locker(&m_lock);
1108  return m_filesize;
1109 }
1110 
1120 {
1121  if (isLocal())
1122  {
1123  return GetFileSize();
1124  }
1125 
1126  QMutexLocker locker(&m_lock);
1127 
1128  if (m_completed ||
1130  {
1131  return m_filesize;
1132  }
1133 
1134  if (!CheckConnection())
1135  {
1136  // Can't establish a new connection, using system one
1137  struct stat fileinfo;
1138 
1139  if (Exists(m_path, &fileinfo))
1140  {
1141  m_filesize = fileinfo.st_size;
1142  }
1143  return m_filesize;
1144  }
1145 
1146  QStringList strlist(m_query.arg(m_recordernum));
1147  strlist << "REQUEST_SIZE";
1148 
1149  bool ok = m_controlSock->SendReceiveStringList(strlist);
1150 
1151  if (ok && !strlist.isEmpty())
1152  {
1153  bool validate;
1154  long long size = strlist[0].toLongLong(&validate);
1155 
1156  if (validate)
1157  {
1158  if (strlist.count() >= 2)
1159  {
1160  m_completed = (strlist[1].toInt() != 0);
1161  }
1162  m_filesize = size;
1163  }
1164  else
1165  {
1166  struct stat fileinfo;
1167 
1168  if (Exists(m_path, &fileinfo))
1169  {
1170  m_filesize = fileinfo.st_size;
1171  }
1172  }
1174  return m_filesize;
1175  }
1176 
1177  return -1;
1178 }
1179 
1180 bool RemoteFile::SaveAs(QByteArray &data)
1181 {
1182  long long fs = GetRealFileSize();
1183 
1184  if (fs < 0)
1185  return false;
1186 
1187  data.resize(fs);
1188  Read(data.data(), fs);
1189 
1190  return true;
1191 }
1192 
1193 void RemoteFile::SetTimeout(bool fast)
1194 {
1195  if (isLocal())
1196  {
1197  // not much we can do with local accesses
1198  return;
1199  }
1200  if (m_timeoutisfast == fast)
1201  return;
1202 
1203  QMutexLocker locker(&m_lock);
1204 
1205  // The m_controlSock variable is valid if the CheckConnection
1206  // function returns true. The local case has already been
1207  // handled. The CheckConnection function can call Resume which
1208  // calls Close, which deletes m_controlSock. However, the
1209  // subsequent call to OpenInternal is guaranteed to recreate the
1210  // socket or return false for a non-local connection, an this must
1211  // be a non-local connection if this line of code is executed.
1212  if (!CheckConnection())
1213  {
1214  LOG(VB_NETWORK, LOG_ERR,
1215  "RemoteFile::SetTimeout(): Couldn't connect");
1216  return;
1217  }
1218 
1219  QStringList strlist( m_query.arg(m_recordernum) );
1220  strlist << "SET_TIMEOUT";
1221  strlist << QString::number((int)fast);
1222 
1224 
1225  m_timeoutisfast = fast;
1226 }
1227 
1228 QDateTime RemoteFile::LastModified(const QString &url)
1229 {
1230  if (isLocal(url))
1231  {
1232  QFileInfo info(url);
1233  return info.lastModified();
1234  }
1235  QDateTime result;
1236  QUrl qurl(url);
1237  QString filename = qurl.path();
1238  QString sgroup = qurl.userName();
1239 
1240  if (!qurl.fragment().isEmpty() || url.endsWith("#"))
1241  filename = filename + "#" + qurl.fragment();
1242 
1243  if (filename.startsWith("/"))
1244  filename = filename.right(filename.length()-1);
1245 
1246  if (filename.isEmpty() || sgroup.isEmpty())
1247  return result;
1248 
1249  QStringList strlist("QUERY_SG_FILEQUERY");
1250  strlist << qurl.host();
1251  strlist << sgroup;
1252  strlist << filename;
1253 
1255 
1256  if (strlist.size() > 1) {
1257 #if QT_VERSION < QT_VERSION_CHECK(5,8,0)
1258  result = MythDate::fromTime_t(strlist[1].toUInt());
1259 #else
1260  if (!strlist[1].isEmpty() && (strlist[1].toInt() != -1))
1261  result = MythDate::fromSecsSinceEpoch(strlist[1].toLongLong());
1262  else
1263  result = QDateTime();;
1264 #endif
1265  }
1266 
1267  return result;
1268 }
1269 
1270 QDateTime RemoteFile::LastModified(void) const
1271 {
1272  return LastModified(m_path);
1273 }
1274 
1284 QString RemoteFile::FindFile(const QString& filename, const QString& host,
1285  const QString& storageGroup, bool useRegex,
1286  bool allowFallback)
1287 {
1288  QStringList files = RemoteFile::FindFileList(filename, host, storageGroup, useRegex, allowFallback);
1289 
1290  if (!files.isEmpty())
1291  return files[0];
1292 
1293  return QString();
1294 }
1295 
1305 QStringList RemoteFile::FindFileList(const QString& filename, const QString& host,
1306  const QString& storageGroup, bool useRegex,
1307  bool allowFallback)
1308 {
1309  LOG(VB_FILE, LOG_INFO, QString("RemoteFile::FindFile(): looking for '%1' on '%2' in group '%3' "
1310  "(useregex: %4, allowfallback: %5)")
1311  .arg(filename).arg(host).arg(storageGroup)
1312  .arg(useRegex).arg(allowFallback));
1313 
1314  if (filename.isEmpty() || storageGroup.isEmpty())
1315  return QStringList();
1316 
1317  QStringList strList;
1318  QString hostName = host;
1319 
1320  if (hostName.isEmpty())
1321  hostName = gCoreContext->GetMasterHostName();
1322 
1323  // if we are looking for the file on this host just search the local storage group first
1324  if (gCoreContext->IsThisBackend(hostName))
1325  {
1326  // We could have made it this far with an IP when we really want
1327  // a hostname
1328  hostName = gCoreContext->GetHostName();
1329  StorageGroup sgroup(storageGroup, hostName);
1330 
1331  if (useRegex)
1332  {
1333  QFileInfo fi(filename);
1334  QStringList files = sgroup.GetFileList('/' + fi.path());
1335 
1336  LOG(VB_FILE, LOG_INFO, QString("RemoteFile::FindFileList: Looking in dir '%1' for '%2'")
1337  .arg(fi.path()).arg(fi.fileName()));
1338 
1339  for (int x = 0; x < files.size(); x++)
1340  {
1341  LOG(VB_FILE, LOG_INFO, QString("RemoteFile::FindFileList: Found '%1 - %2'")
1342  .arg(x).arg(files[x]));
1343  }
1344 
1345  QStringList filteredFiles = files.filter(QRegExp(fi.fileName()));
1346  for (int x = 0; x < filteredFiles.size(); x++)
1347  {
1350  fi.path() + '/' + filteredFiles[x],
1351  storageGroup);
1352  }
1353  }
1354  else
1355  {
1356  if (!sgroup.FindFile(filename).isEmpty())
1357  strList << gCoreContext->GenMythURL(hostName,
1359  filename, storageGroup);
1360  }
1361 
1362  if (!strList.isEmpty() || !allowFallback)
1363  return strList;
1364  }
1365 
1366  // if we didn't find any files ask the master BE to find it
1367  if (strList.isEmpty() && !gCoreContext->IsMasterBackend())
1368  {
1369  strList << "QUERY_FINDFILE" << hostName << storageGroup << filename
1370  << (useRegex ? "1" : "0")
1371  << "1";
1372 
1373  if (gCoreContext->SendReceiveStringList(strList))
1374  {
1375  if (!strList.empty() && !strList[0].isEmpty() &&
1376  strList[0] != "NOT FOUND" && !strList[0].startsWith("ERROR: "))
1377  return strList;
1378  }
1379  }
1380 
1381  return QStringList();
1382 }
1383 
1389 bool RemoteFile::SetBlocking(bool block)
1390 {
1391  if (m_fileWriter)
1392  {
1393  return m_fileWriter->SetBlocking(block);
1394  }
1395  return true;
1396 }
1397 
1405 {
1406  if (IsConnected())
1407  {
1408  return true;
1409  }
1410  if (!m_canresume)
1411  {
1412  return false;
1413  }
1414  return Resume(repos);
1415 }
1416 
1422 {
1423  return m_sock && m_controlSock &&
1425 }
1426 
1432 bool RemoteFile::Resume(bool repos)
1433 {
1434  Close(true);
1435  if (!OpenInternal())
1436  return false;
1437 
1438  if (repos)
1439  {
1441  if (SeekInternal(m_lastposition, SEEK_SET) < 0)
1442  {
1443  Close(true);
1444  LOG(VB_FILE, LOG_ERR,
1445  QString("RemoteFile::Resume: Enable to re-seek into last known "
1446  "position (%1").arg(m_lastposition));
1447  return false;
1448  }
1449  }
1451  return true;
1452 }
1453 
1454 /* vim: set expandtab tabstop=4 shiftwidth=4: */
#define MAX_FILE_CHECK
Definition: remotefile.cpp:37
RemoteFile(const QString &url="", bool write=false, bool usereadahead=true, int timeout_ms=2000, const QStringList *possibleAuxiliaryFiles=nullptr)
Definition: remotefile.cpp:72
def write(text, progress=True)
Definition: mythburn.py:279
static bool DeleteFile(const QString &url)
Definition: remotefile.cpp:418
int restart(void)
Returns milliseconds elapsed since last start() or restart() and resets the count.
Definition: mythtimer.cpp:62
void SetTimeout(bool fast)
bool m_usereadahead
Definition: remotefile.h:79
A QElapsedTimer based timer to replace use of QTime as a timer.
Definition: mythtimer.h:13
void Reset(void)
Definition: remotefile.cpp:746
static bool MoveFile(const QString &src, const QString &dst, bool overwrite=false)
Definition: remotefile.cpp:677
long long SeekInternal(long long pos, int whence, long long curpos=-1)
Definition: remotefile.cpp:766
static void error(const char *str,...)
Definition: vbi.c:42
QString GetBackendServerIP(void)
Returns the IP address of the locally defined backend IP.
bool m_completed
Definition: remotefile.h:94
static QString FindFile(const QString &filename, const QString &host, const QString &storageGroup, bool useRegex=false, bool allowFallback=false)
Search all BE's for a file in the give storage group.
bool m_canresume
Definition: remotefile.h:85
long long m_filesize
Definition: remotefile.h:81
static bool CopyFile(const QString &src, const QString &dst, bool overwrite=false, bool verify=false)
Definition: remotefile.cpp:587
int Read(char *, int size, int max_wait_ms)
Definition: mythsocket.cpp:537
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
bool IsThisBackend(const QString &addr)
is this address mapped to this backend host
long long Seek(long long pos, int whence)
Seek to a position within stream; May be unsafe.
QString m_query
Definition: remotefile.h:91
int m_recordernum
Definition: remotefile.h:86
#define O_LARGEFILE
Definition: remotefile.cpp:19
int m_localFile
Definition: remotefile.h:99
void Close(bool haslock=false)
Definition: remotefile.cpp:375
bool m_timeoutisfast
Definition: remotefile.h:82
int m_timeout_ms
Definition: remotefile.h:80
long long m_lastposition
Definition: remotefile.h:84
bool IsConnected(void)
Check if both the control and data sockets are currently connected.
bool isRunning(void) const
Returns true if start() or restart() has been called at least once since construction and since any c...
Definition: mythtimer.cpp:134
MythSocket * openSocket(bool control)
Definition: remotefile.cpp:121
def read(device=None, features=[])
Definition: disc.py:35
long long GetRealFileSize(void)
GetRealFileSize: returns the current remote file's size.
int GetBackendServerPort(void)
Returns the locally defined backend control port.
QStringList m_auxfiles
Definition: remotefile.h:98
bool OpenInternal(void)
Attempts to resume from a disconnected step.
Definition: remotefile.cpp:268
static bool RemoteSendReceiveStringList(const QString &host, QStringList &strlist)
Definition: remotefile.cpp:39
MythSocket * ConnectCommandSocket(const QString &hostname, int port, const QString &announcement, bool *proto_mismatch=nullptr, int maxConnTry=-1, int setup_timeout=-1)
int Write(const void *data, uint count)
Writes data to the end of the write buffer.
bool SendReceiveStringList(QStringList &strlist, bool quickTimeout=false, bool block=true)
Send a message to the backend and wait for a response.
MBASE_PUBLIC QDateTime fromSecsSinceEpoch(uint seconds)
This function takes the number of seconds since the start of the epoch and returns a QDateTime with t...
Definition: mythdate.cpp:88
#define close
Definition: compat.h:16
bool isLocal(void) const
Definition: remotefile.cpp:116
QString GenMythURL(const QString &host=QString(), int port=0, QString path=QString(), const QString &storageGroup=QString())
int Write(const char *, int size)
Definition: mythsocket.cpp:524
bool IsDataAvailable(void) const
Definition: mythsocket.cpp:567
QString m_path
Definition: remotefile.h:78
void Flush(void)
Allow DiskLoop() to flush buffer completely ignoring low watermark.
virtual int DecrRef(void)
Decrements reference count and deletes on 0.
bool m_writemode
Definition: remotefile.h:93
int Read(void *data, int size)
Definition: remotefile.cpp:936
QString GetMasterHostName(void)
string hostname
Definition: caa.py:17
MythTimer m_lastSizeCheck
Definition: remotefile.h:95
QDateTime LastModified(void) const
#define ENO
This can be appended to the LOG args with "+".
Definition: mythlogging.h:99
bool CheckConnection(bool repos=true)
Check current connection and re-establish it if lost.
ThreadedFileWriter * m_fileWriter
Definition: remotefile.h:100
long long GetFileSize(void) const
GetFileSize: returns the remote file's size at the time it was first opened Will query the server in ...
void Reset(void)
Definition: mythsocket.cpp:551
int elapsed(void) const
Returns milliseconds elapsed since last start() or restart()
Definition: mythtimer.cpp:90
bool Resume(bool repos=true)
Attempts to resume from a disconnected step.
bool ReadStringList(QStringList &list, uint timeoutMS=kShortTimeout)
Definition: mythsocket.cpp:323
long long m_readposition
Definition: remotefile.h:83
static bool Exists(const QString &url, struct stat *fileinfo)
Definition: remotefile.cpp:461
bool SaveAs(QByteArray &data)
bool SetBlocking(bool m_block=true)
Set write blocking mode for the ThreadedFileWriter instance.
bool Open(void)
Opens the file we will be writing to.
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
bool isOpen(void) const
Definition: remotefile.cpp:246
bool ConnectToHost(const QString &hostname, quint16 port)
connect to host
Definition: mythsocket.cpp:384
QString FindFile(const QString &filename)
static QStringList FindFileList(const QString &filename, const QString &host, const QString &storageGroup, bool useRegex=false, bool allowFallback=false)
Search all BE's for files in the give storage group.
bool ReOpen(const QString &newFilename)
Definition: remotefile.cpp:340
bool CheckProtoVersion(MythSocket *socket, uint timeout_ms=kMythSocketLongTimeout, bool error_dialog_desired=false)
bool Open(void)
Definition: remotefile.cpp:255
static const uint kShortTimeout
Definition: mythsocket.h:71
bool IsConnected(void) const
Definition: mythsocket.cpp:561
QStringList m_possibleauxfiles
Definition: remotefile.h:97
This class supports the writing of recordings to disk.
MythSocket * m_controlSock
Definition: remotefile.h:89
Class for communcating between myth backends and frontends.
Definition: mythsocket.h:26
QStringList GetFileList(const QString &Path, bool recursive=false)
bool SendReceiveStringList(QStringList &list, uint min_reply_length=0, uint timeoutMS=kLongTimeout)
Definition: mythsocket.cpp:336
long long Seek(long long pos, int whence, long long curpos=-1)
Definition: remotefile.cpp:759
bool WriteStringList(const QStringList &list)
Definition: mythsocket.cpp:311
MythDB * GetMythDB(void)
Definition: mythdb.cpp:46
bool IsMasterBackend(void)
is this the actual MBE process
int Write(const void *data, int size)
Definition: remotefile.cpp:834
QString FileHash(const QString &filename)
QMutex m_lock
Definition: remotefile.h:88
MythSocket * m_sock
Definition: remotefile.h:90
QString GetHostName(void)
void start(void)
starts measuring elapsed time.
Definition: mythtimer.cpp:47
static QString GetFileHash(const QString &url)
Definition: remotefile.cpp:553
bool SetBlocking(bool block=true)
Set write blocking mode While in blocking mode, ThreadedFileWriter::Write will wait for buffers to be...