MythTV  master
remotefile.cpp
Go to the documentation of this file.
1 #include <iostream>
2 using namespace std;
3 
4 #include <QFile>
5 #include <QFileInfo>
6 #include <QUrl>
7 
8 // POSIX C headers
9 #include <unistd.h>
10 #include <fcntl.h>
11 
12 #include "mythconfig.h"
13 
14 #ifndef O_STREAMING
15 #define O_STREAMING 0
16 #endif
17 
18 #ifndef O_LARGEFILE
19 #define O_LARGEFILE 0
20 #endif
21 
22 #ifndef O_BINARY
23 #define O_BINARY 0
24 #endif
25 
26 #include "mythdb.h"
27 #include "remotefile.h"
28 #include "mythcorecontext.h"
29 #include "mythsocket.h"
30 #include "compat.h"
31 #include "mythtimer.h"
32 #include "mythdate.h"
33 #include "mythmiscutil.h"
34 #include "threadedfilewriter.h"
35 #include "storagegroup.h"
36 
37 #define MAX_FILE_CHECK 500 // in ms
38 
39 static bool RemoteSendReceiveStringList(const QString &host, QStringList &strlist)
40 {
41  bool ok = false;
42 
44  {
45  // since the master backend cannot connect back around to
46  // itself, and the libraries do not have access to the list
47  // of connected slave backends to query an existing connection
48  // start up a new temporary connection directly to the slave
49  // backend to query the file list
50  QString ann = QString("ANN Playback %1 0")
51  .arg(gCoreContext->GetHostName());
52  QString addr = gCoreContext->GetBackendServerIP(host);
53  int port = gCoreContext->GetBackendServerPort(host);
54  bool mismatch = false;
55 
57  addr, port, ann, &mismatch);
58  if (sock)
59  {
60  ok = sock->SendReceiveStringList(strlist);
61  sock->DecrRef();
62  }
63  else
64  strlist.clear();
65  }
66  else
67  ok = gCoreContext->SendReceiveStringList(strlist);
68 
69  return ok;
70 }
71 
72 RemoteFile::RemoteFile(QString url, bool write, bool usereadahead,
73  int timeout_ms,
74  const QStringList *possibleAuxiliaryFiles) :
75  m_path(std::move(url)),
76  m_useReadAhead(usereadahead), m_timeoutMs(timeout_ms),
77  m_writeMode(write)
78 {
79  if (m_writeMode)
80  {
81  m_useReadAhead = false;
82  m_timeoutMs = -1;
83  }
84  else if (possibleAuxiliaryFiles)
85  m_possibleAuxFiles = *possibleAuxiliaryFiles;
86 
87  if (!m_path.isEmpty())
88  Open();
89 
90  LOG(VB_FILE, LOG_DEBUG, QString("RemoteFile(%1)").arg(m_path));
91 }
92 
94 {
95  Close();
96  if (m_controlSock)
97  {
99  m_controlSock = nullptr;
100  }
101  if (m_sock)
102  {
103  m_sock->DecrRef();
104  m_sock = nullptr;
105  }
106 }
107 
108 bool RemoteFile::isLocal(const QString &lpath)
109 {
110  bool is_local = !lpath.isEmpty() &&
111  !lpath.startsWith("myth:") &&
112  (lpath.startsWith("/") || QFile::exists(lpath));
113  return is_local;
114 }
115 
116 bool RemoteFile::isLocal(void) const
117 {
118  return isLocal(m_path);
119 }
120 
122 {
123  QUrl qurl(m_path);
124  QString dir;
125 
126  QString host = qurl.host();
127  int port = qurl.port();
128 
129  dir = qurl.path();
130 
131  if (qurl.hasQuery())
132  dir += "?" + QUrl::fromPercentEncoding(
133  qurl.query(QUrl::FullyEncoded).toLocal8Bit());
134 
135  if (qurl.hasFragment())
136  dir += "#" + qurl.fragment();
137 
138  QString sgroup = qurl.userName();
139 
140  auto *lsock = new MythSocket();
141  QString stype = (control) ? "control socket" : "file data socket";
142 
143  QString loc = QString("RemoteFile::openSocket(%1): ").arg(stype);
144 
145  if (port <= 0)
146  {
147  port = gCoreContext->GetBackendServerPort(host);
148  }
149 
150  if (!lsock->ConnectToHost(host, port))
151  {
152  LOG(VB_GENERAL, LOG_ERR, loc +
153  QString("Could not connect to server %1:%2") .arg(host).arg(port));
154  lsock->DecrRef();
155  return nullptr;
156  }
157 
158  QString hostname = GetMythDB()->GetHostName();
159 
160  QStringList strlist;
161 
162 #ifndef IGNORE_PROTO_VER_MISMATCH
163  if (!gCoreContext->CheckProtoVersion(lsock, 5000))
164  {
165  LOG(VB_GENERAL, LOG_ERR, loc +
166  QString("Failed validation to server %1:%2").arg(host).arg(port));
167  lsock->DecrRef();
168  return nullptr;
169  }
170 #endif
171 
172  if (control)
173  {
174  strlist.append(QString("ANN Playback %1 %2")
175  .arg(hostname).arg(static_cast<int>(false)));
176  if (!lsock->SendReceiveStringList(strlist))
177  {
178  LOG(VB_GENERAL, LOG_ERR, loc +
179  QString("Could not read string list from server %1:%2")
180  .arg(host).arg(port));
181  lsock->DecrRef();
182  return nullptr;
183  }
184  }
185  else
186  {
187  strlist.push_back(QString("ANN FileTransfer %1 %2 %3 %4")
188  .arg(hostname).arg(static_cast<int>(m_writeMode))
189  .arg(static_cast<int>(m_useReadAhead)).arg(m_timeoutMs));
190  strlist << QString("%1").arg(dir);
191  strlist << sgroup;
192 
193  for (const auto& fname : qAsConst(m_possibleAuxFiles))
194  strlist << fname;
195 
196  if (!lsock->SendReceiveStringList(strlist))
197  {
198  LOG(VB_GENERAL, LOG_ERR, loc +
199  QString("Did not get proper response from %1:%2")
200  .arg(host).arg(port));
201  strlist.clear();
202  strlist.push_back("ERROR");
203  strlist.push_back("invalid response");
204  }
205 
206  if (strlist.size() >= 3)
207  {
208  auto it = strlist.begin(); ++it;
209  m_recorderNum = (*it).toInt(); ++it;
210  m_fileSize = (*(it)).toLongLong(); ++it;
211  for (; it != strlist.end(); ++it)
212  m_auxFiles << *it;
213  }
214  else if (!strlist.isEmpty() && strlist.size() < 3 &&
215  strlist[0] != "ERROR")
216  {
217  LOG(VB_GENERAL, LOG_ERR, loc +
218  QString("Did not get proper response from %1:%2")
219  .arg(host).arg(port));
220  strlist.clear();
221  strlist.push_back("ERROR");
222  strlist.push_back("invalid response");
223  }
224  }
225 
226  if (strlist.isEmpty() || strlist[0] == "ERROR")
227  {
228  lsock->DecrRef();
229  lsock = nullptr;
230  if (strlist.isEmpty())
231  {
232  LOG(VB_GENERAL, LOG_ERR, loc + "Failed to open socket, timeout");
233  }
234  else
235  {
236  LOG(VB_GENERAL, LOG_ERR, loc + "Failed to open socket" +
237  ((strlist.size() >= 2) ?
238  QString(", error was %1").arg(strlist[1]) :
239  QString(", remote error")));
240  }
241  }
242 
243  return lsock;
244 }
245 
246 bool RemoteFile::isOpen() const
247 {
248  if (isLocal())
249  {
250  return m_writeMode ? (m_fileWriter != nullptr) : (m_localFile != -1);
251  }
252  return m_sock && m_controlSock;
253 }
254 
256 {
257  if (isOpen())
258  return true;
259 
260  QMutexLocker locker(&m_lock);
261  return OpenInternal();
262 }
263 
269 {
270  if (isLocal())
271  {
272  if (m_writeMode)
273  {
274  // make sure the directories are created if necessary
275  QFileInfo fi(m_path);
276  QDir dir(fi.path());
277  if (!dir.exists())
278  {
279  LOG(VB_FILE, LOG_WARNING, QString("RemoteFile::Open(%1) creating directories")
280  .arg(m_path));
281 
282  if (!dir.mkpath(fi.path()))
283  {
284  LOG(VB_GENERAL, LOG_ERR, QString("RemoteFile::Open(%1) failed to create the directories")
285  .arg(m_path));
286  return false;
287  }
288  }
289 
291  O_WRONLY|O_TRUNC|O_CREAT|O_LARGEFILE,
292  0644);
293 
294  if (!m_fileWriter->Open())
295  {
296  delete m_fileWriter;
297  m_fileWriter = nullptr;
298  LOG(VB_FILE, LOG_ERR, QString("RemoteFile::Open(%1) write mode error")
299  .arg(m_path));
300  return false;
301  }
302  SetBlocking();
303  return true;
304  }
305 
306  // local mode, read only
307  if (!Exists(m_path))
308  {
309  LOG(VB_FILE, LOG_ERR,
310  QString("RemoteFile::Open(%1) Error: Does not exist").arg(m_path));
311  return false;
312  }
313 
314  m_localFile = ::open(m_path.toLocal8Bit().constData(), O_RDONLY);
315  if (m_localFile == -1)
316  {
317  LOG(VB_FILE, LOG_ERR, QString("RemoteFile::Open(%1) Error: %2")
318  .arg(m_path).arg(strerror(errno)));
319  return false;
320  }
321  return true;
322  }
323  m_controlSock = openSocket(true);
324  if (!m_controlSock)
325  return false;
326 
327  m_sock = openSocket(false);
328  if (!m_sock)
329  {
330  // Close the sockets if we received an error so that isOpen() will
331  // return false if the caller tries to use the RemoteFile.
332  Close(true);
333  return false;
334  }
335  m_canResume = true;
336 
337  return true;
338 }
339 
340 bool RemoteFile::ReOpen(const QString& newFilename)
341 {
342  if (isLocal())
343  {
344  if (isOpen())
345  {
346  Close();
347  }
348  m_path = newFilename;
349  return Open();
350  }
351 
352  QMutexLocker locker(&m_lock);
353 
354  if (!CheckConnection(false))
355  {
356  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::ReOpen(): Couldn't connect");
357  return false;
358  }
359 
360  QStringList strlist( m_query.arg(m_recorderNum) );
361  strlist << "REOPEN";
362  strlist << newFilename;
363 
365 
366  m_lock.unlock();
367 
368  bool retval = false;
369  if (!strlist.isEmpty())
370  retval = (strlist[0].toInt() != 0);
371 
372  return retval;
373 }
374 
375 void RemoteFile::Close(bool haslock)
376 {
377  if (isLocal())
378  {
380  m_localFile = -1;
381  delete m_fileWriter;
382  m_fileWriter = nullptr;
383  return;
384  }
385  if (!m_controlSock)
386  return;
387 
388  QStringList strlist( m_query.arg(m_recorderNum) );
389  strlist << "DONE";
390 
391  if (!haslock)
392  {
393  m_lock.lock();
394  }
396  strlist, 0, MythSocket::kShortTimeout))
397  {
398  LOG(VB_GENERAL, LOG_ERR, "Remote file timeout.");
399  }
400 
401  if (m_sock)
402  {
403  m_sock->DecrRef();
404  m_sock = nullptr;
405  }
406  if (m_controlSock)
407  {
409  m_controlSock = nullptr;
410  }
411 
412  if (!haslock)
413  {
414  m_lock.unlock();
415  }
416 }
417 
418 bool RemoteFile::DeleteFile(const QString &url)
419 {
420  if (isLocal(url))
421  {
422  QFile file(url);
423  return file.remove();
424  }
425 
426  bool result = false;
427  QUrl qurl(url);
428  QString filename = qurl.path();
429  QString sgroup = qurl.userName();
430 
431  if (!qurl.fragment().isEmpty() || url.endsWith("#"))
432  filename = filename + "#" + qurl.fragment();
433 
434  if (filename.startsWith("/"))
435  filename = filename.right(filename.length()-1);
436 
437  if (filename.isEmpty() || sgroup.isEmpty())
438  return false;
439 
440  QStringList strlist("DELETE_FILE");
441  strlist << filename;
442  strlist << sgroup;
443 
445 
446  if (!strlist.isEmpty() && strlist[0] == "1")
447  result = true;
448 
449  return result;
450 }
451 
452 bool RemoteFile::Exists(const QString &url)
453 {
454  if (url.isEmpty())
455  return false;
456 
457  struct stat fileinfo {};
458  return Exists(url, &fileinfo);
459 }
460 
461 bool RemoteFile::Exists(const QString &url, struct stat *fileinfo)
462 {
463  if (url.isEmpty())
464  return false;
465 
466  QUrl qurl(url);
467  QString filename = qurl.path();
468  QString sgroup = qurl.userName();
469  QString host = qurl.host();
470 
471  if (isLocal(url) || gCoreContext->IsThisBackend(host))
472  {
473  LOG(VB_FILE, LOG_INFO,
474  QString("RemoteFile::Exists(): looking for local file: %1").arg(url));
475 
476  bool fileExists = false;
477  QString fullFilePath = "";
478 
479  if (url.startsWith("myth:"))
480  {
481  StorageGroup sGroup(sgroup, gCoreContext->GetHostName());
482  fullFilePath = sGroup.FindFile(filename);
483  if (!fullFilePath.isEmpty())
484  fileExists = true;
485  }
486  else
487  {
488  QFileInfo info(url);
489  fileExists = info.exists() /*&& info.isFile()*/;
490  fullFilePath = url;
491  }
492 
493  if (fileExists)
494  {
495  if (stat(fullFilePath.toLocal8Bit().constData(), fileinfo) == -1)
496  {
497  LOG(VB_FILE, LOG_ERR,
498  QString("RemoteFile::Exists(): failed to stat file: %1").arg(fullFilePath) + ENO);
499  }
500  }
501 
502  return fileExists;
503  }
504 
505  LOG(VB_FILE, LOG_INFO,
506  QString("RemoteFile::Exists(): looking for remote file: %1").arg(url));
507 
508  if (!qurl.fragment().isEmpty() || url.endsWith("#"))
509  filename = filename + "#" + qurl.fragment();
510 
511  if (filename.startsWith("/"))
512  filename = filename.right(filename.length()-1);
513 
514  if (filename.isEmpty())
515  return false;
516 
517  QStringList strlist("QUERY_FILE_EXISTS");
518  strlist << filename;
519  if (!sgroup.isEmpty())
520  strlist << sgroup;
521 
522  bool result = false;
523  if (RemoteSendReceiveStringList(host, strlist) && strlist[0] == "1")
524  {
525  if ((strlist.size() >= 15) && fileinfo)
526  {
527  fileinfo->st_dev = strlist[2].toLongLong();
528  fileinfo->st_ino = strlist[3].toLongLong();
529  fileinfo->st_mode = strlist[4].toLongLong();
530  fileinfo->st_nlink = strlist[5].toLongLong();
531  fileinfo->st_uid = strlist[6].toLongLong();
532  fileinfo->st_gid = strlist[7].toLongLong();
533  fileinfo->st_rdev = strlist[8].toLongLong();
534  fileinfo->st_size = strlist[9].toLongLong();
535 #ifndef _WIN32
536  fileinfo->st_blksize = strlist[10].toLongLong();
537  fileinfo->st_blocks = strlist[11].toLongLong();
538 #endif
539  fileinfo->st_atime = strlist[12].toLongLong();
540  fileinfo->st_mtime = strlist[13].toLongLong();
541  fileinfo->st_ctime = strlist[14].toLongLong();
542  result = true;
543  }
544  else if (!fileinfo)
545  {
546  result = true;
547  }
548  }
549 
550  return result;
551 }
552 
553 QString RemoteFile::GetFileHash(const QString &url)
554 {
555  if (isLocal(url))
556  {
557  return FileHash(url);
558  }
559  QString result;
560  QUrl qurl(url);
561  QString filename = qurl.path();
562  QString hostname = qurl.host();
563  QString sgroup = qurl.userName();
564 
565  if (!qurl.fragment().isEmpty() || url.endsWith("#"))
566  filename = filename + "#" + qurl.fragment();
567 
568  if (filename.startsWith("/"))
569  filename = filename.right(filename.length()-1);
570 
571  if (filename.isEmpty() || sgroup.isEmpty())
572  return QString();
573 
574  QStringList strlist("QUERY_FILE_HASH");
575  strlist << filename;
576  strlist << sgroup;
577  strlist << hostname;
578 
580  {
581  result = strlist[0];
582  }
583 
584  return result;
585 }
586 
587 bool RemoteFile::CopyFile (const QString& src, const QString& dst,
588  bool overwrite, bool verify)
589 {
590  LOG(VB_FILE, LOG_INFO,
591  QString("RemoteFile::CopyFile: Copying file from '%1' to '%2'").arg(src).arg(dst));
592 
593  // sanity check
594  if (src == dst)
595  {
596  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::CopyFile: Cannot copy a file to itself");
597  return false;
598  }
599 
600  RemoteFile srcFile(src, false);
601  if (!srcFile.isOpen())
602  {
603  LOG(VB_GENERAL, LOG_ERR,
604  QString("RemoteFile::CopyFile: Failed to open file (%1) for reading.").arg(src));
605  return false;
606  }
607 
608  const int readSize = 2 * 1024 * 1024;
609  char *buf = new char[readSize];
610  if (!buf)
611  {
612  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::CopyFile: ERROR, unable to allocate copy buffer");
613  return false;
614  }
615 
616  if (overwrite)
617  {
618  DeleteFile(dst);
619  }
620  else if (Exists(dst))
621  {
622  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::CopyFile: File already exists");
623  delete[] buf;
624  return false;
625  }
626 
627  RemoteFile dstFile(dst, true);
628  if (!dstFile.isOpen())
629  {
630  LOG(VB_GENERAL, LOG_ERR,
631  QString("RemoteFile::CopyFile: Failed to open file (%1) for writing.").arg(dst));
632  srcFile.Close();
633  delete[] buf;
634  return false;
635  }
636 
637  dstFile.SetBlocking(true);
638 
639  bool success = true;
640  int srcLen = 0;
641 
642  while ((srcLen = srcFile.Read(buf, readSize)) > 0)
643  {
644  int dstLen = dstFile.Write(buf, srcLen);
645 
646  if (dstLen == -1 || srcLen != dstLen)
647  {
648  LOG(VB_GENERAL, LOG_ERR,
649  "RemoteFile::CopyFile: Error while trying to write to destination file.");
650  success = false;
651  }
652  }
653 
654  srcFile.Close();
655  dstFile.Close();
656  delete[] buf;
657 
658  if (success && verify)
659  {
660  // Check written file is correct size
661  struct stat fileinfo {};
662  long long dstSize = Exists(dst, &fileinfo) ? fileinfo.st_size : -1;
663  long long srcSize = srcFile.GetFileSize();
664  if (dstSize != srcSize)
665  {
666  LOG(VB_GENERAL, LOG_ERR,
667  QString("RemoteFile::CopyFile: Copied file is wrong size (%1 rather than %2)")
668  .arg(dstSize).arg(srcSize));
669  success = false;
670  DeleteFile(dst);
671  }
672  }
673 
674  return success;
675 }
676 
677 bool RemoteFile::MoveFile (const QString& src, const QString& dst, bool overwrite)
678 {
679  LOG(VB_FILE, LOG_INFO,
680  QString("RemoteFile::MoveFile: Moving file from '%1' to '%2'").arg(src).arg(dst));
681 
682  // sanity check
683  if (src == dst)
684  {
685  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::MoveFile: Cannot move a file to itself");
686  return false;
687  }
688 
689  if (isLocal(src) != isLocal(dst))
690  {
691  // Moving between local & remote requires a copy & delete
692  bool ok = CopyFile(src, dst, overwrite, true);
693  if (ok)
694  {
695  if (!DeleteFile(src))
696  LOG(VB_FILE, LOG_ERR,
697  "RemoteFile::MoveFile: Failed to delete file after successful copy");
698  }
699  return ok;
700  }
701 
702  if (overwrite)
703  {
704  DeleteFile(dst);
705  }
706  else if (Exists(dst))
707  {
708  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::MoveFile: File already exists");
709  return false;
710  }
711 
712  if (isLocal(src))
713  {
714  // Moving local -> local
715  QFileInfo fi(dst);
716  if (QDir().mkpath(fi.path()) && QFile::rename(src, dst))
717  return true;
718 
719  LOG(VB_FILE, LOG_ERR, "RemoteFile::MoveFile: Rename failed");
720  return false;
721  }
722 
723  // Moving remote -> remote
724  QUrl srcUrl(src);
725  QUrl dstUrl(dst);
726 
727  if (srcUrl.userName() != dstUrl.userName())
728  {
729  LOG(VB_FILE, LOG_ERR, "RemoteFile::MoveFile: Cannot change a file's Storage Group");
730  return false;
731  }
732 
733  QStringList strlist("MOVE_FILE");
734  strlist << srcUrl.userName() << srcUrl.path() << dstUrl.path();
735 
737 
738  if (!strlist.isEmpty() && strlist[0] == "1")
739  return true;
740 
741  LOG(VB_FILE, LOG_ERR, QString("RemoteFile::MoveFile: MOVE_FILE failed with: %1")
742  .arg(strlist.join(",")));
743  return false;
744 }
745 
747 {
748  if (isLocal())
749  return;
750  QMutexLocker locker(&m_lock);
751  if (!m_sock)
752  {
753  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::Reset(): Called with no socket");
754  return;
755  }
756  m_sock->Reset();
757 }
758 
759 long long RemoteFile::Seek(long long pos, int whence, long long curpos)
760 {
761  QMutexLocker locker(&m_lock);
762 
763  return SeekInternal(pos, whence, curpos);
764 }
765 
766 long long RemoteFile::SeekInternal(long long pos, int whence, long long curpos)
767 {
768  if (isLocal())
769  {
770  if (!isOpen())
771  {
772  LOG(VB_FILE, LOG_ERR, "RemoteFile::Seek(): Called with no file opened");
773  return -1;
774  }
775  if (m_writeMode)
776  return m_fileWriter->Seek(pos, whence);
777 
778  long long offset = 0LL;
779  if (whence == SEEK_SET)
780  {
781  QFileInfo info(m_path);
782  offset = min(pos, info.size());
783  }
784  else if (whence == SEEK_END)
785  {
786  QFileInfo info(m_path);
787  offset = info.size() + pos;
788  }
789  else if (whence == SEEK_CUR)
790  {
791  offset = ((curpos > 0) ? curpos : ::lseek64(m_localFile, 0, SEEK_CUR)) + pos;
792  }
793  else
794  return -1;
795 
796  off64_t localpos = ::lseek64(m_localFile, pos, whence);
797  if (localpos != pos)
798  {
799  LOG(VB_FILE, LOG_ERR,
800  QString("RemoteFile::Seek(): Couldn't seek to offset %1")
801  .arg(offset));
802  return -1;
803  }
804  return localpos;
805  }
806 
807  if (!CheckConnection(false))
808  {
809  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::Seek(): Couldn't connect");
810  return -1;
811  }
812 
813  QStringList strlist( m_query.arg(m_recorderNum) );
814  strlist << "SEEK";
815  strlist << QString::number(pos);
816  strlist << QString::number(whence);
817  if (curpos > 0)
818  strlist << QString::number(curpos);
819  else
820  strlist << QString::number(m_readPosition);
821 
822  bool ok = m_controlSock->SendReceiveStringList(strlist);
823 
824  if (ok && !strlist.isEmpty())
825  {
826  m_lastPosition = m_readPosition = strlist[0].toLongLong();
827  m_sock->Reset();
828  return strlist[0].toLongLong();
829  }
830  m_lastPosition = 0LL;
831  return -1;
832 }
833 
834 int RemoteFile::Write(const void *data, int size)
835 {
836  int recv = 0;
837  int sent = 0;
838  unsigned zerocnt = 0;
839  bool error = false;
840  bool response = false;
841 
842  if (!m_writeMode)
843  {
844  LOG(VB_NETWORK, LOG_ERR,
845  "RemoteFile::Write(): Called when not in write mode");
846  return -1;
847  }
848  if (isLocal())
849  {
850  if (!isOpen())
851  {
852  LOG(VB_FILE, LOG_ERR,
853  "RemoteFile::Write(): File not opened");
854  return -1;
855  }
856  return m_fileWriter->Write(data, size);
857  }
858 
859  QMutexLocker locker(&m_lock);
860 
861  if (!CheckConnection())
862  {
863  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::Write(): Couldn't connect");
864  return -1;
865  }
866 
867  QStringList strlist( m_query.arg(m_recorderNum) );
868  strlist << "WRITE_BLOCK";
869  strlist << QString::number(size);
870  bool ok = m_controlSock->WriteStringList(strlist);
871  if (!ok)
872  {
873  LOG(VB_NETWORK, LOG_ERR,
874  "RemoteFile::Write(): Block notification failed");
875  return -1;
876  }
877 
878  recv = size;
879  while (sent < recv && !error && zerocnt++ < 50)
880  {
881  int ret = m_sock->Write((char*)data + sent, recv - sent);
882  if (ret > 0)
883  {
884  sent += ret;
885  }
886  else
887  {
888  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::Write(): socket error");
889  error = true;
890  break;
891  }
892 
895  !strlist.isEmpty())
896  {
897  recv = strlist[0].toInt(); // -1 on backend error
898  response = true;
899  }
900  }
901 
902  if (!error && !response)
903  {
905  !strlist.isEmpty())
906  {
907  recv = strlist[0].toInt(); // -1 on backend error
908  }
909  else
910  {
911  LOG(VB_GENERAL, LOG_ERR,
912  "RemoteFile::Write(): No response from control socket.");
913  recv = -1;
914  }
915  }
916 
917  LOG(VB_NETWORK, LOG_DEBUG,
918  QString("RemoteFile::Write(): reqd=%1, sent=%2, rept=%3, error=%4")
919  .arg(size).arg(sent).arg(recv).arg(error));
920 
921  if (recv < 0)
922  return recv;
923 
924  if (error || recv != sent)
925  {
926  sent = -1;
927  }
928  else
929  {
930  m_lastPosition += sent;
931  }
932 
933  return sent;
934 }
935 
936 int RemoteFile::Read(void *data, int size)
937 {
938  int recv = 0;
939  int sent = 0;
940  bool error = false;
941  bool response = false;
942 
943  QMutexLocker locker(&m_lock);
944 
945  if (isLocal())
946  {
947  if (m_writeMode)
948  {
949  LOG(VB_FILE, LOG_ERR, "RemoteFile:Read() called in writing mode");
950  return -1;
951  }
952  if (isOpen())
953  {
954  return ::read(m_localFile, data, size);
955  }
956  LOG(VB_FILE, LOG_ERR, "RemoteFile:Read() called when local file not opened");
957  return -1;
958  }
959 
960  if (!CheckConnection())
961  {
962  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::Read(): Couldn't connect");
963  return -1;
964  }
965 
966  if (m_sock->IsDataAvailable())
967  {
968  LOG(VB_NETWORK, LOG_ERR,
969  "RemoteFile::Read(): Read socket not empty to start!");
970  m_sock->Reset();
971  }
972 
973  while (m_controlSock->IsDataAvailable())
974  {
975  LOG(VB_NETWORK, LOG_WARNING,
976  "RemoteFile::Read(): Control socket not empty to start!");
977  m_controlSock->Reset();
978  }
979 
980  QStringList strlist( m_query.arg(m_recorderNum) );
981  strlist << "REQUEST_BLOCK";
982  strlist << QString::number(size);
983  bool ok = m_controlSock->WriteStringList(strlist);
984  if (!ok)
985  {
986  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::Read(): Block request failed");
987  return -1;
988  }
989 
990  sent = size;
991 
992  int waitms = 30;
993  MythTimer mtimer;
994  mtimer.start();
995 
996  while (recv < sent && !error && mtimer.elapsed() < 10000)
997  {
998  int ret = m_sock->Read(((char *)data) + recv, sent - recv, waitms);
999 
1000  if (ret > 0)
1001  recv += ret;
1002  else if (ret < 0)
1003  error = true;
1004 
1005  waitms += (waitms < 200) ? 20 : 0;
1006 
1007  if (m_controlSock->IsDataAvailable() &&
1009  !strlist.isEmpty())
1010  {
1011  sent = strlist[0].toInt(); // -1 on backend error
1012  response = true;
1013  if (ret < sent)
1014  {
1015  // We have received less than what the server sent, retry immediately
1016  ret = m_sock->Read(((char *)data) + recv, sent - recv, waitms);
1017  if (ret > 0)
1018  recv += ret;
1019  else if (ret < 0)
1020  error = true;
1021  }
1022  }
1023  }
1024 
1025  if (!error && !response)
1026  {
1027  // Wait up to 1.5s for the backend to send the size
1028  // MythSocket::ReadString will drop the connection
1029  if (m_controlSock->ReadStringList(strlist, 1500) &&
1030  !strlist.isEmpty())
1031  {
1032  sent = strlist[0].toInt(); // -1 on backend error
1033  }
1034  else
1035  {
1036  LOG(VB_GENERAL, LOG_ERR,
1037  "RemoteFile::Read(): No response from control socket.");
1038  // If no data was received from control socket, and we got what we asked for
1039  // assume everything is okay
1040  if (recv == size)
1041  {
1042  sent = recv;
1043  }
1044  else
1045  {
1046  sent = -1;
1047  }
1048  // The TCP socket is dropped if there's a timeout, so we reconnect
1049  if (!Resume())
1050  {
1051  sent = -1;
1052  }
1053  }
1054  }
1055 
1056  LOG(VB_NETWORK, LOG_DEBUG,
1057  QString("Read(): reqd=%1, rcvd=%2, rept=%3, error=%4")
1058  .arg(size).arg(recv).arg(sent).arg(error));
1059 
1060  if (sent < 0)
1061  return sent;
1062 
1063  if (error || sent != recv)
1064  {
1065  LOG(VB_GENERAL, LOG_WARNING,
1066  QString("RemoteFile::Read(): sent %1 != recv %2")
1067  .arg(sent).arg(recv));
1068  recv = -1;
1069 
1070  // The TCP socket is dropped if there's a timeout, so we reconnect
1071  if (!Resume())
1072  {
1073  LOG(VB_GENERAL, LOG_WARNING, "RemoteFile::Read(): Resume failed.");
1074  }
1075  else
1076  LOG(VB_GENERAL, LOG_NOTICE, "RemoteFile::Read(): Resume success.");
1077  }
1078  else
1079  {
1080  m_lastPosition += recv;
1081  }
1082 
1083  return recv;
1084 }
1085 
1091 long long RemoteFile::GetFileSize(void) const
1092 {
1093  if (isLocal())
1094  {
1095  if (isOpen() && m_writeMode)
1096  {
1097  m_fileWriter->Flush();
1098  }
1099  if (Exists(m_path))
1100  {
1101  QFileInfo info(m_path);
1102  return info.size();
1103  }
1104  return -1;
1105  }
1106 
1107  QMutexLocker locker(&m_lock);
1108  return m_fileSize;
1109 }
1110 
1120 {
1121  if (isLocal())
1122  {
1123  return GetFileSize();
1124  }
1125 
1126  QMutexLocker locker(&m_lock);
1127 
1128  if (m_completed ||
1130  {
1131  return m_fileSize;
1132  }
1133 
1134  if (!CheckConnection())
1135  {
1136  // Can't establish a new connection, using system one
1137  struct stat fileinfo {};
1138 
1139  if (Exists(m_path, &fileinfo))
1140  {
1141  m_fileSize = fileinfo.st_size;
1142  }
1143  return m_fileSize;
1144  }
1145 
1146  QStringList strlist(m_query.arg(m_recorderNum));
1147  strlist << "REQUEST_SIZE";
1148 
1149  bool ok = m_controlSock->SendReceiveStringList(strlist);
1150 
1151  if (ok && !strlist.isEmpty())
1152  {
1153  bool validate = false;
1154  long long size = strlist[0].toLongLong(&validate);
1155 
1156  if (validate)
1157  {
1158  if (strlist.count() >= 2)
1159  {
1160  m_completed = (strlist[1].toInt() != 0);
1161  }
1162  m_fileSize = size;
1163  }
1164  else
1165  {
1166  struct stat fileinfo {};
1167 
1168  if (Exists(m_path, &fileinfo))
1169  {
1170  m_fileSize = fileinfo.st_size;
1171  }
1172  }
1174  return m_fileSize;
1175  }
1176 
1177  return -1;
1178 }
1179 
1180 bool RemoteFile::SaveAs(QByteArray &data)
1181 {
1182  long long fs = GetRealFileSize();
1183 
1184  if (fs < 0)
1185  return false;
1186 
1187  data.resize(fs);
1188  Read(data.data(), fs);
1189 
1190  return true;
1191 }
1192 
1193 void RemoteFile::SetTimeout(bool fast)
1194 {
1195  if (isLocal())
1196  {
1197  // not much we can do with local accesses
1198  return;
1199  }
1200  if (m_timeoutIsFast == fast)
1201  return;
1202 
1203  QMutexLocker locker(&m_lock);
1204 
1205  // The m_controlSock variable is valid if the CheckConnection
1206  // function returns true. The local case has already been
1207  // handled. The CheckConnection function can call Resume which
1208  // calls Close, which deletes m_controlSock. However, the
1209  // subsequent call to OpenInternal is guaranteed to recreate the
1210  // socket or return false for a non-local connection, and this must
1211  // be a non-local connection if this line of code is executed.
1212  if (!CheckConnection())
1213  {
1214  LOG(VB_NETWORK, LOG_ERR,
1215  "RemoteFile::SetTimeout(): Couldn't connect");
1216  return;
1217  }
1218  if (m_controlSock == nullptr)
1219  return;
1220 
1221  QStringList strlist( m_query.arg(m_recorderNum) );
1222  strlist << "SET_TIMEOUT";
1223  strlist << QString::number((int)fast);
1224 
1226 
1227  m_timeoutIsFast = fast;
1228 }
1229 
1230 QDateTime RemoteFile::LastModified(const QString &url)
1231 {
1232  if (isLocal(url))
1233  {
1234  QFileInfo info(url);
1235  return info.lastModified();
1236  }
1237  QDateTime result;
1238  QUrl qurl(url);
1239  QString filename = qurl.path();
1240  QString sgroup = qurl.userName();
1241 
1242  if (!qurl.fragment().isEmpty() || url.endsWith("#"))
1243  filename = filename + "#" + qurl.fragment();
1244 
1245  if (filename.startsWith("/"))
1246  filename = filename.right(filename.length()-1);
1247 
1248  if (filename.isEmpty() || sgroup.isEmpty())
1249  return result;
1250 
1251  QStringList strlist("QUERY_SG_FILEQUERY");
1252  strlist << qurl.host();
1253  strlist << sgroup;
1254  strlist << filename;
1255 
1257 
1258  if (strlist.size() > 1) {
1259 #if QT_VERSION < QT_VERSION_CHECK(5,8,0)
1260  result = MythDate::fromTime_t(strlist[1].toUInt());
1261 #else
1262  if (!strlist[1].isEmpty() && (strlist[1].toInt() != -1))
1263  result = MythDate::fromSecsSinceEpoch(strlist[1].toLongLong());
1264  else
1265  result = QDateTime();;
1266 #endif
1267  }
1268 
1269  return result;
1270 }
1271 
1272 QDateTime RemoteFile::LastModified(void) const
1273 {
1274  return LastModified(m_path);
1275 }
1276 
1286 QString RemoteFile::FindFile(const QString& filename, const QString& host,
1287  const QString& storageGroup, bool useRegex,
1288  bool allowFallback)
1289 {
1290  QStringList files = RemoteFile::FindFileList(filename, host, storageGroup, useRegex, allowFallback);
1291 
1292  if (!files.isEmpty())
1293  return files[0];
1294 
1295  return QString();
1296 }
1297 
1307 QStringList RemoteFile::FindFileList(const QString& filename, const QString& host,
1308  const QString& storageGroup, bool useRegex,
1309  bool allowFallback)
1310 {
1311  LOG(VB_FILE, LOG_INFO, QString("RemoteFile::FindFile(): looking for '%1' on '%2' in group '%3' "
1312  "(useregex: %4, allowfallback: %5)")
1313  .arg(filename).arg(host).arg(storageGroup)
1314  .arg(useRegex).arg(allowFallback));
1315 
1316  if (filename.isEmpty() || storageGroup.isEmpty())
1317  return QStringList();
1318 
1319  QStringList strList;
1320  QString hostName = host;
1321 
1322  if (hostName.isEmpty())
1323  hostName = gCoreContext->GetMasterHostName();
1324 
1325  // if we are looking for the file on this host just search the local storage group first
1326  if (gCoreContext->IsThisBackend(hostName))
1327  {
1328  // We could have made it this far with an IP when we really want
1329  // a hostname
1330  hostName = gCoreContext->GetHostName();
1331  StorageGroup sgroup(storageGroup, hostName);
1332 
1333  if (useRegex)
1334  {
1335  QFileInfo fi(filename);
1336  QStringList files = sgroup.GetFileList('/' + fi.path());
1337 
1338  LOG(VB_FILE, LOG_INFO, QString("RemoteFile::FindFileList: Looking in dir '%1' for '%2'")
1339  .arg(fi.path()).arg(fi.fileName()));
1340 
1341  for (int x = 0; x < files.size(); x++)
1342  {
1343  LOG(VB_FILE, LOG_INFO, QString("RemoteFile::FindFileList: Found '%1 - %2'")
1344  .arg(x).arg(files[x]));
1345  }
1346 
1347  QStringList filteredFiles = files.filter(QRegExp(fi.fileName()));
1348  for (int x = 0; x < filteredFiles.size(); x++)
1349  {
1352  fi.path() + '/' + filteredFiles[x],
1353  storageGroup);
1354  }
1355  }
1356  else
1357  {
1358  if (!sgroup.FindFile(filename).isEmpty())
1359  {
1360  strList << MythCoreContext::GenMythURL(hostName,
1362  filename, storageGroup);
1363  }
1364  }
1365 
1366  if (!strList.isEmpty() || !allowFallback)
1367  return strList;
1368  }
1369 
1370  // if we didn't find any files ask the master BE to find it
1371  if (strList.isEmpty() && !gCoreContext->IsMasterBackend())
1372  {
1373  strList << "QUERY_FINDFILE" << hostName << storageGroup << filename
1374  << (useRegex ? "1" : "0")
1375  << "1";
1376 
1377  if (gCoreContext->SendReceiveStringList(strList))
1378  {
1379  if (!strList.empty() && !strList[0].isEmpty() &&
1380  strList[0] != "NOT FOUND" && !strList[0].startsWith("ERROR: "))
1381  return strList;
1382  }
1383  }
1384 
1385  return QStringList();
1386 }
1387 
1393 bool RemoteFile::SetBlocking(bool block)
1394 {
1395  if (m_fileWriter)
1396  {
1397  return m_fileWriter->SetBlocking(block);
1398  }
1399  return true;
1400 }
1401 
1409 {
1410  if (IsConnected())
1411  {
1412  return true;
1413  }
1414  if (!m_canResume)
1415  {
1416  return false;
1417  }
1418  return Resume(repos);
1419 }
1420 
1426 {
1427  return m_sock && m_controlSock &&
1429 }
1430 
1436 bool RemoteFile::Resume(bool repos)
1437 {
1438  Close(true);
1439  if (!OpenInternal())
1440  return false;
1441 
1442  if (repos)
1443  {
1445  if (SeekInternal(m_lastPosition, SEEK_SET) < 0)
1446  {
1447  Close(true);
1448  LOG(VB_FILE, LOG_ERR,
1449  QString("RemoteFile::Resume: Enable to re-seek into last known "
1450  "position (%1").arg(m_lastPosition));
1451  return false;
1452  }
1453  }
1455  return true;
1456 }
1457 
1458 /* vim: set expandtab tabstop=4 shiftwidth=4: */
QStringList m_auxFiles
Definition: remotefile.h:98
#define MAX_FILE_CHECK
Definition: remotefile.cpp:37
def write(text, progress=True)
Definition: mythburn.py:308
long long m_readPosition
Definition: remotefile.h:83
static bool DeleteFile(const QString &url)
Definition: remotefile.cpp:418
int restart(void)
Returns milliseconds elapsed since last start() or restart() and resets the count.
Definition: mythtimer.cpp:62
void SetTimeout(bool fast)
A QElapsedTimer based timer to replace use of QTime as a timer.
Definition: mythtimer.h:13
void Reset(void)
Definition: remotefile.cpp:746
static void error(const char *str,...)
Definition: vbi.cpp:42
static bool MoveFile(const QString &src, const QString &dst, bool overwrite=false)
Definition: remotefile.cpp:677
long long SeekInternal(long long pos, int whence, long long curpos=-1)
Definition: remotefile.cpp:766
QString GetBackendServerIP(void)
Returns the IP address of the locally defined backend IP.
long long m_fileSize
Definition: remotefile.h:81
bool m_completed
Definition: remotefile.h:94
bool m_timeoutIsFast
Definition: remotefile.h:82
static QString FindFile(const QString &filename, const QString &host, const QString &storageGroup, bool useRegex=false, bool allowFallback=false)
Search all BE's for a file in the give storage group.
long long m_lastPosition
Definition: remotefile.h:84
bool IsDataAvailable(void)
Definition: mythsocket.cpp:567
int m_recorderNum
Definition: remotefile.h:86
QStringList m_possibleAuxFiles
Definition: remotefile.h:97
int Write(const char *data, int size)
Definition: mythsocket.cpp:524
static bool CopyFile(const QString &src, const QString &dst, bool overwrite=false, bool verify=false)
Definition: remotefile.cpp:587
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
bool IsThisBackend(const QString &addr)
is this address mapped to this backend host
long long Seek(long long pos, int whence)
Seek to a position within stream; May be unsafe.
bool m_useReadAhead
Definition: remotefile.h:79
QString m_query
Definition: remotefile.h:91
#define O_LARGEFILE
Definition: remotefile.cpp:19
int m_localFile
Definition: remotefile.h:99
void Close(bool haslock=false)
Definition: remotefile.cpp:375
bool IsConnected(void)
Check if both the control and data sockets are currently connected.
bool isRunning(void) const
Returns true if start() or restart() has been called at least once since construction and since any c...
Definition: mythtimer.cpp:134
MythSocket * openSocket(bool control)
Definition: remotefile.cpp:121
bool m_writeMode
Definition: remotefile.h:93
int Read(char *data, int size, int max_wait_ms)
Definition: mythsocket.cpp:537
def read(device=None, features=[])
Definition: disc.py:35
long long GetRealFileSize(void)
GetRealFileSize: returns the current remote file's size.
int GetBackendServerPort(void)
Returns the locally defined backend control port.
bool OpenInternal(void)
Attempts to resume from a disconnected step.
Definition: remotefile.cpp:268
static bool RemoteSendReceiveStringList(const QString &host, QStringList &strlist)
Definition: remotefile.cpp:39
MythSocket * ConnectCommandSocket(const QString &hostname, int port, const QString &announcement, bool *proto_mismatch=nullptr, int maxConnTry=-1, int setup_timeout=-1)
int Write(const void *data, uint count)
Writes data to the end of the write buffer.
bool SendReceiveStringList(QStringList &strlist, bool quickTimeout=false, bool block=true)
Send a message to the backend and wait for a response.
MBASE_PUBLIC QDateTime fromSecsSinceEpoch(uint seconds)
This function takes the number of seconds since the start of the epoch and returns a QDateTime with t...
Definition: mythdate.cpp:88
#define close
Definition: compat.h:16
bool isLocal(void) const
Definition: remotefile.cpp:116
static QString GenMythURL(const QString &host=QString(), int port=0, QString path=QString(), const QString &storageGroup=QString())
QString m_path
Definition: remotefile.h:78
void Flush(void)
Allow DiskLoop() to flush buffer completely ignoring low watermark.
virtual int DecrRef(void)
Decrements reference count and deletes on 0.
int m_timeoutMs
Definition: remotefile.h:80
int Read(void *data, int size)
Definition: remotefile.cpp:936
QString GetMasterHostName(void)
string hostname
Definition: caa.py:17
MythTimer m_lastSizeCheck
Definition: remotefile.h:95
QDateTime LastModified(void) const
RemoteFile(QString url="", bool write=false, bool usereadahead=true, int timeout_ms=2000, const QStringList *possibleAuxiliaryFiles=nullptr)
Definition: remotefile.cpp:72
#define ENO
This can be appended to the LOG args with "+".
Definition: mythlogging.h:69
bool CheckConnection(bool repos=true)
Check current connection and re-establish it if lost.
ThreadedFileWriter * m_fileWriter
Definition: remotefile.h:100
long long GetFileSize(void) const
GetFileSize: returns the remote file's size at the time it was first opened Will query the server in ...
void Reset(void)
Definition: mythsocket.cpp:551
bool Resume(bool repos=true)
Attempts to resume from a disconnected step.
bool ReadStringList(QStringList &list, uint timeoutMS=kShortTimeout)
Definition: mythsocket.cpp:323
static bool Exists(const QString &url, struct stat *fileinfo)
Definition: remotefile.cpp:461
bool SaveAs(QByteArray &data)
bool SetBlocking(bool m_block=true)
Set write blocking mode for the ThreadedFileWriter instance.
bool Open(void)
Opens the file we will be writing to.
bool isOpen(void) const
Definition: remotefile.cpp:246
QString FindFile(const QString &filename)
int elapsed(void)
Returns milliseconds elapsed since last start() or restart()
Definition: mythtimer.cpp:90
static QStringList FindFileList(const QString &filename, const QString &host, const QString &storageGroup, bool useRegex=false, bool allowFallback=false)
Search all BE's for files in the give storage group.
bool ReOpen(const QString &newFilename)
Definition: remotefile.cpp:340
bool CheckProtoVersion(MythSocket *socket, uint timeout_ms=kMythSocketLongTimeout, bool error_dialog_desired=false)
bool Open(void)
Definition: remotefile.cpp:255
static const uint kShortTimeout
Definition: mythsocket.h:71
bool IsConnected(void) const
Definition: mythsocket.cpp:561
This class supports the writing of recordings to disk.
MythSocket * m_controlSock
Definition: remotefile.h:89
Class for communcating between myth backends and frontends.
Definition: mythsocket.h:26
QStringList GetFileList(const QString &Path, bool recursive=false)
bool SendReceiveStringList(QStringList &list, uint min_reply_length=0, uint timeoutMS=kLongTimeout)
Definition: mythsocket.cpp:336
long long Seek(long long pos, int whence, long long curpos=-1)
Definition: remotefile.cpp:759
bool WriteStringList(const QStringList &list)
Definition: mythsocket.cpp:311
MythDB * GetMythDB(void)
Definition: mythdb.cpp:46
bool IsMasterBackend(void)
is this the actual MBE process
bool m_canResume
Definition: remotefile.h:85
int Write(const void *data, int size)
Definition: remotefile.cpp:834
QString FileHash(const QString &filename)
QMutex m_lock
Definition: remotefile.h:88
MythSocket * m_sock
Definition: remotefile.h:90
QString GetHostName(void)
void start(void)
starts measuring elapsed time.
Definition: mythtimer.cpp:47
static QString GetFileHash(const QString &url)
Definition: remotefile.cpp:553
bool SetBlocking(bool block=true)
Set write blocking mode While in blocking mode, ThreadedFileWriter::Write will wait for buffers to be...
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:23