MythTV  master
remotefile.cpp
Go to the documentation of this file.
1 #include <iostream>
2 using namespace std;
3 
4 #include <QFile>
5 #include <QFileInfo>
6 #include <QUrl>
7 
8 // POSIX C headers
9 #include <unistd.h>
10 #include <fcntl.h>
11 
12 #include "mythconfig.h"
13 
14 #ifndef O_STREAMING
15 #define O_STREAMING 0
16 #endif
17 
18 #ifndef O_LARGEFILE
19 #define O_LARGEFILE 0
20 #endif
21 
22 #ifndef O_BINARY
23 #define O_BINARY 0
24 #endif
25 
26 #include "mythdb.h"
27 #include "remotefile.h"
28 #include "mythcorecontext.h"
29 #include "mythsocket.h"
30 #include "compat.h"
31 #include "mythtimer.h"
32 #include "mythdate.h"
33 #include "mythmiscutil.h"
34 #include "threadedfilewriter.h"
35 #include "storagegroup.h"
36 
37 #define MAX_FILE_CHECK 500 // in ms
38 
39 static bool RemoteSendReceiveStringList(const QString &host, QStringList &strlist)
40 {
41  bool ok = false;
42 
44  {
45  // since the master backend cannot connect back around to
46  // itself, and the libraries do not have access to the list
47  // of connected slave backends to query an existing connection
48  // start up a new temporary connection directly to the slave
49  // backend to query the file list
50  QString ann = QString("ANN Playback %1 0")
51  .arg(gCoreContext->GetHostName());
52  QString addr = gCoreContext->GetBackendServerIP(host);
53  int port = gCoreContext->GetBackendServerPort(host);
54  bool mismatch = false;
55 
57  addr, port, ann, &mismatch);
58  if (sock)
59  {
60  ok = sock->SendReceiveStringList(strlist);
61  sock->DecrRef();
62  }
63  else
64  strlist.clear();
65  }
66  else
67  ok = gCoreContext->SendReceiveStringList(strlist);
68 
69  return ok;
70 }
71 
72 RemoteFile::RemoteFile(QString url, bool write, bool usereadahead,
73  int timeout_ms,
74  const QStringList *possibleAuxiliaryFiles) :
75  m_path(std::move(url)),
76  m_useReadAhead(usereadahead), m_timeoutMs(timeout_ms),
77  m_writeMode(write)
78 {
79  if (m_writeMode)
80  {
81  m_useReadAhead = false;
82  m_timeoutMs = -1;
83  }
84  else if (possibleAuxiliaryFiles)
85  m_possibleAuxFiles = *possibleAuxiliaryFiles;
86 
87  if (!m_path.isEmpty())
88  Open();
89 
90  LOG(VB_FILE, LOG_DEBUG, QString("RemoteFile(%1)").arg(m_path));
91 }
92 
94 {
95  Close();
96  if (m_controlSock)
97  {
99  m_controlSock = nullptr;
100  }
101  if (m_sock)
102  {
103  m_sock->DecrRef();
104  m_sock = nullptr;
105  }
106 }
107 
108 bool RemoteFile::isLocal(const QString &lpath)
109 {
110  bool is_local = !lpath.isEmpty() &&
111  !lpath.startsWith("myth:") &&
112  (lpath.startsWith("/") || QFile::exists(lpath));
113  return is_local;
114 }
115 
116 bool RemoteFile::isLocal(void) const
117 {
118  return isLocal(m_path);
119 }
120 
122 {
123  QUrl qurl(m_path);
124  QString dir;
125 
126  QString host = qurl.host();
127  int port = qurl.port();
128 
129  dir = qurl.path();
130 
131  if (qurl.hasQuery())
132  dir += "?" + QUrl::fromPercentEncoding(
133  qurl.query(QUrl::FullyEncoded).toLocal8Bit());
134 
135  if (qurl.hasFragment())
136  dir += "#" + qurl.fragment();
137 
138  QString sgroup = qurl.userName();
139 
140  auto *lsock = new MythSocket();
141  QString stype = (control) ? "control socket" : "file data socket";
142 
143  QString loc = QString("RemoteFile::openSocket(%1): ").arg(stype);
144 
145  if (port <= 0)
146  {
147  port = gCoreContext->GetBackendServerPort(host);
148  }
149 
150  if (!lsock->ConnectToHost(host, port))
151  {
152  LOG(VB_GENERAL, LOG_ERR, loc +
153  QString("Could not connect to server %1:%2") .arg(host).arg(port));
154  lsock->DecrRef();
155  return nullptr;
156  }
157 
158  QString hostname = GetMythDB()->GetHostName();
159 
160  QStringList strlist;
161 
162 #ifndef IGNORE_PROTO_VER_MISMATCH
163  if (!gCoreContext->CheckProtoVersion(lsock, 5000))
164  {
165  LOG(VB_GENERAL, LOG_ERR, loc +
166  QString("Failed validation to server %1:%2").arg(host).arg(port));
167  lsock->DecrRef();
168  return nullptr;
169  }
170 #endif
171 
172  if (control)
173  {
174  strlist.append(QString("ANN Playback %1 %2").arg(hostname).arg(false));
175  if (!lsock->SendReceiveStringList(strlist))
176  {
177  LOG(VB_GENERAL, LOG_ERR, loc +
178  QString("Could not read string list from server %1:%2")
179  .arg(host).arg(port));
180  lsock->DecrRef();
181  return nullptr;
182  }
183  }
184  else
185  {
186  strlist.push_back(QString("ANN FileTransfer %1 %2 %3 %4")
187  .arg(hostname).arg(m_writeMode)
188  .arg(m_useReadAhead).arg(m_timeoutMs));
189  strlist << QString("%1").arg(dir);
190  strlist << sgroup;
191 
192  foreach (auto fname, m_possibleAuxFiles)
193  strlist << fname;
194 
195  if (!lsock->SendReceiveStringList(strlist))
196  {
197  LOG(VB_GENERAL, LOG_ERR, loc +
198  QString("Did not get proper response from %1:%2")
199  .arg(host).arg(port));
200  strlist.clear();
201  strlist.push_back("ERROR");
202  strlist.push_back("invalid response");
203  }
204 
205  if (strlist.size() >= 3)
206  {
207  auto it = strlist.begin(); ++it;
208  m_recorderNum = (*it).toInt(); ++it;
209  m_fileSize = (*(it)).toLongLong(); ++it;
210  for (; it != strlist.end(); ++it)
211  m_auxFiles << *it;
212  }
213  else if (!strlist.isEmpty() && strlist.size() < 3 &&
214  strlist[0] != "ERROR")
215  {
216  LOG(VB_GENERAL, LOG_ERR, loc +
217  QString("Did not get proper response from %1:%2")
218  .arg(host).arg(port));
219  strlist.clear();
220  strlist.push_back("ERROR");
221  strlist.push_back("invalid response");
222  }
223  }
224 
225  if (strlist.isEmpty() || strlist[0] == "ERROR")
226  {
227  lsock->DecrRef();
228  lsock = nullptr;
229  if (strlist.isEmpty())
230  {
231  LOG(VB_GENERAL, LOG_ERR, loc + "Failed to open socket, timeout");
232  }
233  else
234  {
235  LOG(VB_GENERAL, LOG_ERR, loc + "Failed to open socket" +
236  ((strlist.size() >= 2) ?
237  QString(", error was %1").arg(strlist[1]) :
238  QString(", remote error")));
239  }
240  }
241 
242  return lsock;
243 }
244 
245 bool RemoteFile::isOpen() const
246 {
247  if (isLocal())
248  {
249  return m_writeMode ? (m_fileWriter != nullptr) : (m_localFile != -1);
250  }
251  return m_sock && m_controlSock;
252 }
253 
255 {
256  if (isOpen())
257  return true;
258 
259  QMutexLocker locker(&m_lock);
260  return OpenInternal();
261 }
262 
268 {
269  if (isLocal())
270  {
271  if (m_writeMode)
272  {
273  // make sure the directories are created if necessary
274  QFileInfo fi(m_path);
275  QDir dir(fi.path());
276  if (!dir.exists())
277  {
278  LOG(VB_FILE, LOG_WARNING, QString("RemoteFile::Open(%1) creating directories")
279  .arg(m_path));
280 
281  if (!dir.mkpath(fi.path()))
282  {
283  LOG(VB_GENERAL, LOG_ERR, QString("RemoteFile::Open(%1) failed to create the directories")
284  .arg(m_path));
285  return false;
286  }
287  }
288 
290  O_WRONLY|O_TRUNC|O_CREAT|O_LARGEFILE,
291  0644);
292 
293  if (!m_fileWriter->Open())
294  {
295  delete m_fileWriter;
296  m_fileWriter = nullptr;
297  LOG(VB_FILE, LOG_ERR, QString("RemoteFile::Open(%1) write mode error")
298  .arg(m_path));
299  return false;
300  }
301  SetBlocking();
302  return true;
303  }
304 
305  // local mode, read only
306  if (!Exists(m_path))
307  {
308  LOG(VB_FILE, LOG_ERR,
309  QString("RemoteFile::Open(%1) Error: Does not exist").arg(m_path));
310  return false;
311  }
312 
313  m_localFile = ::open(m_path.toLocal8Bit().constData(), O_RDONLY);
314  if (m_localFile == -1)
315  {
316  LOG(VB_FILE, LOG_ERR, QString("RemoteFile::Open(%1) Error: %2")
317  .arg(m_path).arg(strerror(errno)));
318  return false;
319  }
320  return true;
321  }
322  m_controlSock = openSocket(true);
323  if (!m_controlSock)
324  return false;
325 
326  m_sock = openSocket(false);
327  if (!m_sock)
328  {
329  // Close the sockets if we received an error so that isOpen() will
330  // return false if the caller tries to use the RemoteFile.
331  Close(true);
332  return false;
333  }
334  m_canResume = true;
335 
336  return true;
337 }
338 
339 bool RemoteFile::ReOpen(const QString& newFilename)
340 {
341  if (isLocal())
342  {
343  if (isOpen())
344  {
345  Close();
346  }
347  m_path = newFilename;
348  return Open();
349  }
350 
351  QMutexLocker locker(&m_lock);
352 
353  if (!CheckConnection(false))
354  {
355  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::ReOpen(): Couldn't connect");
356  return false;
357  }
358 
359  QStringList strlist( m_query.arg(m_recorderNum) );
360  strlist << "REOPEN";
361  strlist << newFilename;
362 
364 
365  m_lock.unlock();
366 
367  bool retval = false;
368  if (!strlist.isEmpty())
369  retval = (strlist[0].toInt() != 0);
370 
371  return retval;
372 }
373 
374 void RemoteFile::Close(bool haslock)
375 {
376  if (isLocal())
377  {
379  m_localFile = -1;
380  delete m_fileWriter;
381  m_fileWriter = nullptr;
382  return;
383  }
384  if (!m_controlSock)
385  return;
386 
387  QStringList strlist( m_query.arg(m_recorderNum) );
388  strlist << "DONE";
389 
390  if (!haslock)
391  {
392  m_lock.lock();
393  }
395  strlist, 0, MythSocket::kShortTimeout))
396  {
397  LOG(VB_GENERAL, LOG_ERR, "Remote file timeout.");
398  }
399 
400  if (m_sock)
401  {
402  m_sock->DecrRef();
403  m_sock = nullptr;
404  }
405  if (m_controlSock)
406  {
408  m_controlSock = nullptr;
409  }
410 
411  if (!haslock)
412  {
413  m_lock.unlock();
414  }
415 }
416 
417 bool RemoteFile::DeleteFile(const QString &url)
418 {
419  if (isLocal(url))
420  {
421  QFile file(url);
422  return file.remove();
423  }
424 
425  bool result = false;
426  QUrl qurl(url);
427  QString filename = qurl.path();
428  QString sgroup = qurl.userName();
429 
430  if (!qurl.fragment().isEmpty() || url.endsWith("#"))
431  filename = filename + "#" + qurl.fragment();
432 
433  if (filename.startsWith("/"))
434  filename = filename.right(filename.length()-1);
435 
436  if (filename.isEmpty() || sgroup.isEmpty())
437  return false;
438 
439  QStringList strlist("DELETE_FILE");
440  strlist << filename;
441  strlist << sgroup;
442 
444 
445  if (!strlist.isEmpty() && strlist[0] == "1")
446  result = true;
447 
448  return result;
449 }
450 
451 bool RemoteFile::Exists(const QString &url)
452 {
453  if (url.isEmpty())
454  return false;
455 
456  struct stat fileinfo {};
457  return Exists(url, &fileinfo);
458 }
459 
460 bool RemoteFile::Exists(const QString &url, struct stat *fileinfo)
461 {
462  if (url.isEmpty())
463  return false;
464 
465  QUrl qurl(url);
466  QString filename = qurl.path();
467  QString sgroup = qurl.userName();
468  QString host = qurl.host();
469 
470  if (isLocal(url) || gCoreContext->IsThisBackend(host))
471  {
472  LOG(VB_FILE, LOG_INFO,
473  QString("RemoteFile::Exists(): looking for local file: %1").arg(url));
474 
475  bool fileExists = false;
476  QString fullFilePath = "";
477 
478  if (url.startsWith("myth:"))
479  {
480  StorageGroup sGroup(sgroup, gCoreContext->GetHostName());
481  fullFilePath = sGroup.FindFile(filename);
482  if (!fullFilePath.isEmpty())
483  fileExists = true;
484  }
485  else
486  {
487  QFileInfo info(url);
488  fileExists = info.exists() /*&& info.isFile()*/;
489  fullFilePath = url;
490  }
491 
492  if (fileExists)
493  {
494  if (stat(fullFilePath.toLocal8Bit().constData(), fileinfo) == -1)
495  {
496  LOG(VB_FILE, LOG_ERR,
497  QString("RemoteFile::Exists(): failed to stat file: %1").arg(fullFilePath) + ENO);
498  }
499  }
500 
501  return fileExists;
502  }
503 
504  LOG(VB_FILE, LOG_INFO,
505  QString("RemoteFile::Exists(): looking for remote file: %1").arg(url));
506 
507  if (!qurl.fragment().isEmpty() || url.endsWith("#"))
508  filename = filename + "#" + qurl.fragment();
509 
510  if (filename.startsWith("/"))
511  filename = filename.right(filename.length()-1);
512 
513  if (filename.isEmpty())
514  return false;
515 
516  QStringList strlist("QUERY_FILE_EXISTS");
517  strlist << filename;
518  if (!sgroup.isEmpty())
519  strlist << sgroup;
520 
521  bool result = false;
522  if (RemoteSendReceiveStringList(host, strlist) && strlist[0] == "1")
523  {
524  if ((strlist.size() >= 15) && fileinfo)
525  {
526  fileinfo->st_dev = strlist[2].toLongLong();
527  fileinfo->st_ino = strlist[3].toLongLong();
528  fileinfo->st_mode = strlist[4].toLongLong();
529  fileinfo->st_nlink = strlist[5].toLongLong();
530  fileinfo->st_uid = strlist[6].toLongLong();
531  fileinfo->st_gid = strlist[7].toLongLong();
532  fileinfo->st_rdev = strlist[8].toLongLong();
533  fileinfo->st_size = strlist[9].toLongLong();
534 #ifndef _WIN32
535  fileinfo->st_blksize = strlist[10].toLongLong();
536  fileinfo->st_blocks = strlist[11].toLongLong();
537 #endif
538  fileinfo->st_atime = strlist[12].toLongLong();
539  fileinfo->st_mtime = strlist[13].toLongLong();
540  fileinfo->st_ctime = strlist[14].toLongLong();
541  result = true;
542  }
543  else if (!fileinfo)
544  {
545  result = true;
546  }
547  }
548 
549  return result;
550 }
551 
552 QString RemoteFile::GetFileHash(const QString &url)
553 {
554  if (isLocal(url))
555  {
556  return FileHash(url);
557  }
558  QString result;
559  QUrl qurl(url);
560  QString filename = qurl.path();
561  QString hostname = qurl.host();
562  QString sgroup = qurl.userName();
563 
564  if (!qurl.fragment().isEmpty() || url.endsWith("#"))
565  filename = filename + "#" + qurl.fragment();
566 
567  if (filename.startsWith("/"))
568  filename = filename.right(filename.length()-1);
569 
570  if (filename.isEmpty() || sgroup.isEmpty())
571  return QString();
572 
573  QStringList strlist("QUERY_FILE_HASH");
574  strlist << filename;
575  strlist << sgroup;
576  strlist << hostname;
577 
579  {
580  result = strlist[0];
581  }
582 
583  return result;
584 }
585 
586 bool RemoteFile::CopyFile (const QString& src, const QString& dst,
587  bool overwrite, bool verify)
588 {
589  LOG(VB_FILE, LOG_INFO,
590  QString("RemoteFile::CopyFile: Copying file from '%1' to '%2'").arg(src).arg(dst));
591 
592  // sanity check
593  if (src == dst)
594  {
595  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::CopyFile: Cannot copy a file to itself");
596  return false;
597  }
598 
599  RemoteFile srcFile(src, false);
600  if (!srcFile.isOpen())
601  {
602  LOG(VB_GENERAL, LOG_ERR,
603  QString("RemoteFile::CopyFile: Failed to open file (%1) for reading.").arg(src));
604  return false;
605  }
606 
607  const int readSize = 2 * 1024 * 1024;
608  char *buf = new char[readSize];
609  if (!buf)
610  {
611  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::CopyFile: ERROR, unable to allocate copy buffer");
612  return false;
613  }
614 
615  if (overwrite)
616  {
617  DeleteFile(dst);
618  }
619  else if (Exists(dst))
620  {
621  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::CopyFile: File already exists");
622  delete[] buf;
623  return false;
624  }
625 
626  RemoteFile dstFile(dst, true);
627  if (!dstFile.isOpen())
628  {
629  LOG(VB_GENERAL, LOG_ERR,
630  QString("RemoteFile::CopyFile: Failed to open file (%1) for writing.").arg(dst));
631  srcFile.Close();
632  delete[] buf;
633  return false;
634  }
635 
636  dstFile.SetBlocking(true);
637 
638  bool success = true;
639  int srcLen = 0;
640 
641  while ((srcLen = srcFile.Read(buf, readSize)) > 0)
642  {
643  int dstLen = dstFile.Write(buf, srcLen);
644 
645  if (dstLen == -1 || srcLen != dstLen)
646  {
647  LOG(VB_GENERAL, LOG_ERR,
648  "RemoteFile::CopyFile: Error while trying to write to destination file.");
649  success = false;
650  }
651  }
652 
653  srcFile.Close();
654  dstFile.Close();
655  delete[] buf;
656 
657  if (success && verify)
658  {
659  // Check written file is correct size
660  struct stat fileinfo {};
661  long long dstSize = Exists(dst, &fileinfo) ? fileinfo.st_size : -1;
662  long long srcSize = srcFile.GetFileSize();
663  if (dstSize != srcSize)
664  {
665  LOG(VB_GENERAL, LOG_ERR,
666  QString("RemoteFile::CopyFile: Copied file is wrong size (%1 rather than %2)")
667  .arg(dstSize).arg(srcSize));
668  success = false;
669  DeleteFile(dst);
670  }
671  }
672 
673  return success;
674 }
675 
676 bool RemoteFile::MoveFile (const QString& src, const QString& dst, bool overwrite)
677 {
678  LOG(VB_FILE, LOG_INFO,
679  QString("RemoteFile::MoveFile: Moving file from '%1' to '%2'").arg(src).arg(dst));
680 
681  // sanity check
682  if (src == dst)
683  {
684  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::MoveFile: Cannot move a file to itself");
685  return false;
686  }
687 
688  if (isLocal(src) != isLocal(dst))
689  {
690  // Moving between local & remote requires a copy & delete
691  bool ok = CopyFile(src, dst, overwrite, true);
692  if (ok)
693  {
694  if (!DeleteFile(src))
695  LOG(VB_FILE, LOG_ERR,
696  "RemoteFile::MoveFile: Failed to delete file after successful copy");
697  }
698  return ok;
699  }
700 
701  if (overwrite)
702  {
703  DeleteFile(dst);
704  }
705  else if (Exists(dst))
706  {
707  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::MoveFile: File already exists");
708  return false;
709  }
710 
711  if (isLocal(src))
712  {
713  // Moving local -> local
714  QFileInfo fi(dst);
715  if (QDir().mkpath(fi.path()) && QFile::rename(src, dst))
716  return true;
717 
718  LOG(VB_FILE, LOG_ERR, "RemoteFile::MoveFile: Rename failed");
719  return false;
720  }
721 
722  // Moving remote -> remote
723  QUrl srcUrl(src);
724  QUrl dstUrl(dst);
725 
726  if (srcUrl.userName() != dstUrl.userName())
727  {
728  LOG(VB_FILE, LOG_ERR, "RemoteFile::MoveFile: Cannot change a file's Storage Group");
729  return false;
730  }
731 
732  QStringList strlist("MOVE_FILE");
733  strlist << srcUrl.userName() << srcUrl.path() << dstUrl.path();
734 
736 
737  if (!strlist.isEmpty() && strlist[0] == "1")
738  return true;
739 
740  LOG(VB_FILE, LOG_ERR, QString("RemoteFile::MoveFile: MOVE_FILE failed with: %1")
741  .arg(strlist.join(",")));
742  return false;
743 }
744 
746 {
747  if (isLocal())
748  return;
749  QMutexLocker locker(&m_lock);
750  if (!m_sock)
751  {
752  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::Reset(): Called with no socket");
753  return;
754  }
755  m_sock->Reset();
756 }
757 
758 long long RemoteFile::Seek(long long pos, int whence, long long curpos)
759 {
760  QMutexLocker locker(&m_lock);
761 
762  return SeekInternal(pos, whence, curpos);
763 }
764 
765 long long RemoteFile::SeekInternal(long long pos, int whence, long long curpos)
766 {
767  if (isLocal())
768  {
769  if (!isOpen())
770  {
771  LOG(VB_FILE, LOG_ERR, "RemoteFile::Seek(): Called with no file opened");
772  return -1;
773  }
774  if (m_writeMode)
775  return m_fileWriter->Seek(pos, whence);
776 
777  long long offset = 0LL;
778  if (whence == SEEK_SET)
779  {
780  QFileInfo info(m_path);
781  offset = min(pos, info.size());
782  }
783  else if (whence == SEEK_END)
784  {
785  QFileInfo info(m_path);
786  offset = info.size() + pos;
787  }
788  else if (whence == SEEK_CUR)
789  {
790  offset = ((curpos > 0) ? curpos : ::lseek64(m_localFile, 0, SEEK_CUR)) + pos;
791  }
792  else
793  return -1;
794 
795  off64_t localpos = ::lseek64(m_localFile, pos, whence);
796  if (localpos != pos)
797  {
798  LOG(VB_FILE, LOG_ERR,
799  QString("RemoteFile::Seek(): Couldn't seek to offset %1")
800  .arg(offset));
801  return -1;
802  }
803  return localpos;
804  }
805 
806  if (!CheckConnection(false))
807  {
808  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::Seek(): Couldn't connect");
809  return -1;
810  }
811 
812  QStringList strlist( m_query.arg(m_recorderNum) );
813  strlist << "SEEK";
814  strlist << QString::number(pos);
815  strlist << QString::number(whence);
816  if (curpos > 0)
817  strlist << QString::number(curpos);
818  else
819  strlist << QString::number(m_readPosition);
820 
821  bool ok = m_controlSock->SendReceiveStringList(strlist);
822 
823  if (ok && !strlist.isEmpty())
824  {
825  m_lastPosition = m_readPosition = strlist[0].toLongLong();
826  m_sock->Reset();
827  return strlist[0].toLongLong();
828  }
829  m_lastPosition = 0LL;
830  return -1;
831 }
832 
833 int RemoteFile::Write(const void *data, int size)
834 {
835  int recv = 0;
836  int sent = 0;
837  unsigned zerocnt = 0;
838  bool error = false;
839  bool response = false;
840 
841  if (!m_writeMode)
842  {
843  LOG(VB_NETWORK, LOG_ERR,
844  "RemoteFile::Write(): Called when not in write mode");
845  return -1;
846  }
847  if (isLocal())
848  {
849  if (!isOpen())
850  {
851  LOG(VB_FILE, LOG_ERR,
852  "RemoteFile::Write(): File not opened");
853  return -1;
854  }
855  return m_fileWriter->Write(data, size);
856  }
857 
858  QMutexLocker locker(&m_lock);
859 
860  if (!CheckConnection())
861  {
862  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::Write(): Couldn't connect");
863  return -1;
864  }
865 
866  QStringList strlist( m_query.arg(m_recorderNum) );
867  strlist << "WRITE_BLOCK";
868  strlist << QString::number(size);
869  bool ok = m_controlSock->WriteStringList(strlist);
870  if (!ok)
871  {
872  LOG(VB_NETWORK, LOG_ERR,
873  "RemoteFile::Write(): Block notification failed");
874  return -1;
875  }
876 
877  recv = size;
878  while (sent < recv && !error && zerocnt++ < 50)
879  {
880  int ret = m_sock->Write((char*)data + sent, recv - sent);
881  if (ret > 0)
882  {
883  sent += ret;
884  }
885  else
886  {
887  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::Write(): socket error");
888  error = true;
889  break;
890  }
891 
894  !strlist.isEmpty())
895  {
896  recv = strlist[0].toInt(); // -1 on backend error
897  response = true;
898  }
899  }
900 
901  if (!error && !response)
902  {
904  !strlist.isEmpty())
905  {
906  recv = strlist[0].toInt(); // -1 on backend error
907  }
908  else
909  {
910  LOG(VB_GENERAL, LOG_ERR,
911  "RemoteFile::Write(): No response from control socket.");
912  recv = -1;
913  }
914  }
915 
916  LOG(VB_NETWORK, LOG_DEBUG,
917  QString("RemoteFile::Write(): reqd=%1, sent=%2, rept=%3, error=%4")
918  .arg(size).arg(sent).arg(recv).arg(error));
919 
920  if (recv < 0)
921  return recv;
922 
923  if (error || recv != sent)
924  {
925  sent = -1;
926  }
927  else
928  {
929  m_lastPosition += sent;
930  }
931 
932  return sent;
933 }
934 
935 int RemoteFile::Read(void *data, int size)
936 {
937  int recv = 0;
938  int sent = 0;
939  bool error = false;
940  bool response = false;
941 
942  QMutexLocker locker(&m_lock);
943 
944  if (isLocal())
945  {
946  if (m_writeMode)
947  {
948  LOG(VB_FILE, LOG_ERR, "RemoteFile:Read() called in writing mode");
949  return -1;
950  }
951  if (isOpen())
952  {
953  return ::read(m_localFile, data, size);
954  }
955  LOG(VB_FILE, LOG_ERR, "RemoteFile:Read() called when local file not opened");
956  return -1;
957  }
958 
959  if (!CheckConnection())
960  {
961  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::Read(): Couldn't connect");
962  return -1;
963  }
964 
965  if (m_sock->IsDataAvailable())
966  {
967  LOG(VB_NETWORK, LOG_ERR,
968  "RemoteFile::Read(): Read socket not empty to start!");
969  m_sock->Reset();
970  }
971 
972  while (m_controlSock->IsDataAvailable())
973  {
974  LOG(VB_NETWORK, LOG_WARNING,
975  "RemoteFile::Read(): Control socket not empty to start!");
976  m_controlSock->Reset();
977  }
978 
979  QStringList strlist( m_query.arg(m_recorderNum) );
980  strlist << "REQUEST_BLOCK";
981  strlist << QString::number(size);
982  bool ok = m_controlSock->WriteStringList(strlist);
983  if (!ok)
984  {
985  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::Read(): Block request failed");
986  return -1;
987  }
988 
989  sent = size;
990 
991  int waitms = 30;
992  MythTimer mtimer;
993  mtimer.start();
994 
995  while (recv < sent && !error && mtimer.elapsed() < 10000)
996  {
997  int ret = m_sock->Read(((char *)data) + recv, sent - recv, waitms);
998 
999  if (ret > 0)
1000  recv += ret;
1001  else if (ret < 0)
1002  error = true;
1003 
1004  waitms += (waitms < 200) ? 20 : 0;
1005 
1006  if (m_controlSock->IsDataAvailable() &&
1008  !strlist.isEmpty())
1009  {
1010  sent = strlist[0].toInt(); // -1 on backend error
1011  response = true;
1012  if (ret < sent)
1013  {
1014  // We have received less than what the server sent, retry immediately
1015  ret = m_sock->Read(((char *)data) + recv, sent - recv, waitms);
1016  if (ret > 0)
1017  recv += ret;
1018  else if (ret < 0)
1019  error = true;
1020  }
1021  }
1022  }
1023 
1024  if (!error && !response)
1025  {
1026  // Wait up to 1.5s for the backend to send the size
1027  // MythSocket::ReadString will drop the connection
1028  if (m_controlSock->ReadStringList(strlist, 1500) &&
1029  !strlist.isEmpty())
1030  {
1031  sent = strlist[0].toInt(); // -1 on backend error
1032  }
1033  else
1034  {
1035  LOG(VB_GENERAL, LOG_ERR,
1036  "RemoteFile::Read(): No response from control socket.");
1037  // If no data was received from control socket, and we got what we asked for
1038  // assume everything is okay
1039  if (recv == size)
1040  {
1041  sent = recv;
1042  }
1043  else
1044  {
1045  sent = -1;
1046  }
1047  // The TCP socket is dropped if there's a timeout, so we reconnect
1048  if (!Resume())
1049  {
1050  sent = -1;
1051  }
1052  }
1053  }
1054 
1055  LOG(VB_NETWORK, LOG_DEBUG,
1056  QString("Read(): reqd=%1, rcvd=%2, rept=%3, error=%4")
1057  .arg(size).arg(recv).arg(sent).arg(error));
1058 
1059  if (sent < 0)
1060  return sent;
1061 
1062  if (error || sent != recv)
1063  {
1064  LOG(VB_GENERAL, LOG_WARNING,
1065  QString("RemoteFile::Read(): sent %1 != recv %2")
1066  .arg(sent).arg(recv));
1067  recv = -1;
1068 
1069  // The TCP socket is dropped if there's a timeout, so we reconnect
1070  if (!Resume())
1071  {
1072  LOG(VB_GENERAL, LOG_WARNING, "RemoteFile::Read(): Resume failed.");
1073  }
1074  else
1075  LOG(VB_GENERAL, LOG_NOTICE, "RemoteFile::Read(): Resume success.");
1076  }
1077  else
1078  {
1079  m_lastPosition += recv;
1080  }
1081 
1082  return recv;
1083 }
1084 
1090 long long RemoteFile::GetFileSize(void) const
1091 {
1092  if (isLocal())
1093  {
1094  if (isOpen() && m_writeMode)
1095  {
1096  m_fileWriter->Flush();
1097  }
1098  if (Exists(m_path))
1099  {
1100  QFileInfo info(m_path);
1101  return info.size();
1102  }
1103  return -1;
1104  }
1105 
1106  QMutexLocker locker(&m_lock);
1107  return m_fileSize;
1108 }
1109 
1119 {
1120  if (isLocal())
1121  {
1122  return GetFileSize();
1123  }
1124 
1125  QMutexLocker locker(&m_lock);
1126 
1127  if (m_completed ||
1129  {
1130  return m_fileSize;
1131  }
1132 
1133  if (!CheckConnection())
1134  {
1135  // Can't establish a new connection, using system one
1136  struct stat fileinfo {};
1137 
1138  if (Exists(m_path, &fileinfo))
1139  {
1140  m_fileSize = fileinfo.st_size;
1141  }
1142  return m_fileSize;
1143  }
1144 
1145  QStringList strlist(m_query.arg(m_recorderNum));
1146  strlist << "REQUEST_SIZE";
1147 
1148  bool ok = m_controlSock->SendReceiveStringList(strlist);
1149 
1150  if (ok && !strlist.isEmpty())
1151  {
1152  bool validate = false;
1153  long long size = strlist[0].toLongLong(&validate);
1154 
1155  if (validate)
1156  {
1157  if (strlist.count() >= 2)
1158  {
1159  m_completed = (strlist[1].toInt() != 0);
1160  }
1161  m_fileSize = size;
1162  }
1163  else
1164  {
1165  struct stat fileinfo {};
1166 
1167  if (Exists(m_path, &fileinfo))
1168  {
1169  m_fileSize = fileinfo.st_size;
1170  }
1171  }
1173  return m_fileSize;
1174  }
1175 
1176  return -1;
1177 }
1178 
1179 bool RemoteFile::SaveAs(QByteArray &data)
1180 {
1181  long long fs = GetRealFileSize();
1182 
1183  if (fs < 0)
1184  return false;
1185 
1186  data.resize(fs);
1187  Read(data.data(), fs);
1188 
1189  return true;
1190 }
1191 
1192 void RemoteFile::SetTimeout(bool fast)
1193 {
1194  if (isLocal())
1195  {
1196  // not much we can do with local accesses
1197  return;
1198  }
1199  if (m_timeoutIsFast == fast)
1200  return;
1201 
1202  QMutexLocker locker(&m_lock);
1203 
1204  // The m_controlSock variable is valid if the CheckConnection
1205  // function returns true. The local case has already been
1206  // handled. The CheckConnection function can call Resume which
1207  // calls Close, which deletes m_controlSock. However, the
1208  // subsequent call to OpenInternal is guaranteed to recreate the
1209  // socket or return false for a non-local connection, an this must
1210  // be a non-local connection if this line of code is executed.
1211  if (!CheckConnection())
1212  {
1213  LOG(VB_NETWORK, LOG_ERR,
1214  "RemoteFile::SetTimeout(): Couldn't connect");
1215  return;
1216  }
1217 
1218  QStringList strlist( m_query.arg(m_recorderNum) );
1219  strlist << "SET_TIMEOUT";
1220  strlist << QString::number((int)fast);
1221 
1223 
1224  m_timeoutIsFast = fast;
1225 }
1226 
1227 QDateTime RemoteFile::LastModified(const QString &url)
1228 {
1229  if (isLocal(url))
1230  {
1231  QFileInfo info(url);
1232  return info.lastModified();
1233  }
1234  QDateTime result;
1235  QUrl qurl(url);
1236  QString filename = qurl.path();
1237  QString sgroup = qurl.userName();
1238 
1239  if (!qurl.fragment().isEmpty() || url.endsWith("#"))
1240  filename = filename + "#" + qurl.fragment();
1241 
1242  if (filename.startsWith("/"))
1243  filename = filename.right(filename.length()-1);
1244 
1245  if (filename.isEmpty() || sgroup.isEmpty())
1246  return result;
1247 
1248  QStringList strlist("QUERY_SG_FILEQUERY");
1249  strlist << qurl.host();
1250  strlist << sgroup;
1251  strlist << filename;
1252 
1254 
1255  if (strlist.size() > 1) {
1256 #if QT_VERSION < QT_VERSION_CHECK(5,8,0)
1257  result = MythDate::fromTime_t(strlist[1].toUInt());
1258 #else
1259  if (!strlist[1].isEmpty() && (strlist[1].toInt() != -1))
1260  result = MythDate::fromSecsSinceEpoch(strlist[1].toLongLong());
1261  else
1262  result = QDateTime();;
1263 #endif
1264  }
1265 
1266  return result;
1267 }
1268 
1269 QDateTime RemoteFile::LastModified(void) const
1270 {
1271  return LastModified(m_path);
1272 }
1273 
1283 QString RemoteFile::FindFile(const QString& filename, const QString& host,
1284  const QString& storageGroup, bool useRegex,
1285  bool allowFallback)
1286 {
1287  QStringList files = RemoteFile::FindFileList(filename, host, storageGroup, useRegex, allowFallback);
1288 
1289  if (!files.isEmpty())
1290  return files[0];
1291 
1292  return QString();
1293 }
1294 
1304 QStringList RemoteFile::FindFileList(const QString& filename, const QString& host,
1305  const QString& storageGroup, bool useRegex,
1306  bool allowFallback)
1307 {
1308  LOG(VB_FILE, LOG_INFO, QString("RemoteFile::FindFile(): looking for '%1' on '%2' in group '%3' "
1309  "(useregex: %4, allowfallback: %5)")
1310  .arg(filename).arg(host).arg(storageGroup)
1311  .arg(useRegex).arg(allowFallback));
1312 
1313  if (filename.isEmpty() || storageGroup.isEmpty())
1314  return QStringList();
1315 
1316  QStringList strList;
1317  QString hostName = host;
1318 
1319  if (hostName.isEmpty())
1320  hostName = gCoreContext->GetMasterHostName();
1321 
1322  // if we are looking for the file on this host just search the local storage group first
1323  if (gCoreContext->IsThisBackend(hostName))
1324  {
1325  // We could have made it this far with an IP when we really want
1326  // a hostname
1327  hostName = gCoreContext->GetHostName();
1328  StorageGroup sgroup(storageGroup, hostName);
1329 
1330  if (useRegex)
1331  {
1332  QFileInfo fi(filename);
1333  QStringList files = sgroup.GetFileList('/' + fi.path());
1334 
1335  LOG(VB_FILE, LOG_INFO, QString("RemoteFile::FindFileList: Looking in dir '%1' for '%2'")
1336  .arg(fi.path()).arg(fi.fileName()));
1337 
1338  for (int x = 0; x < files.size(); x++)
1339  {
1340  LOG(VB_FILE, LOG_INFO, QString("RemoteFile::FindFileList: Found '%1 - %2'")
1341  .arg(x).arg(files[x]));
1342  }
1343 
1344  QStringList filteredFiles = files.filter(QRegExp(fi.fileName()));
1345  for (int x = 0; x < filteredFiles.size(); x++)
1346  {
1349  fi.path() + '/' + filteredFiles[x],
1350  storageGroup);
1351  }
1352  }
1353  else
1354  {
1355  if (!sgroup.FindFile(filename).isEmpty())
1356  {
1357  strList << MythCoreContext::GenMythURL(hostName,
1359  filename, storageGroup);
1360  }
1361  }
1362 
1363  if (!strList.isEmpty() || !allowFallback)
1364  return strList;
1365  }
1366 
1367  // if we didn't find any files ask the master BE to find it
1368  if (strList.isEmpty() && !gCoreContext->IsMasterBackend())
1369  {
1370  strList << "QUERY_FINDFILE" << hostName << storageGroup << filename
1371  << (useRegex ? "1" : "0")
1372  << "1";
1373 
1374  if (gCoreContext->SendReceiveStringList(strList))
1375  {
1376  if (!strList.empty() && !strList[0].isEmpty() &&
1377  strList[0] != "NOT FOUND" && !strList[0].startsWith("ERROR: "))
1378  return strList;
1379  }
1380  }
1381 
1382  return QStringList();
1383 }
1384 
1390 bool RemoteFile::SetBlocking(bool block)
1391 {
1392  if (m_fileWriter)
1393  {
1394  return m_fileWriter->SetBlocking(block);
1395  }
1396  return true;
1397 }
1398 
1406 {
1407  if (IsConnected())
1408  {
1409  return true;
1410  }
1411  if (!m_canResume)
1412  {
1413  return false;
1414  }
1415  return Resume(repos);
1416 }
1417 
1423 {
1424  return m_sock && m_controlSock &&
1426 }
1427 
1433 bool RemoteFile::Resume(bool repos)
1434 {
1435  Close(true);
1436  if (!OpenInternal())
1437  return false;
1438 
1439  if (repos)
1440  {
1442  if (SeekInternal(m_lastPosition, SEEK_SET) < 0)
1443  {
1444  Close(true);
1445  LOG(VB_FILE, LOG_ERR,
1446  QString("RemoteFile::Resume: Enable to re-seek into last known "
1447  "position (%1").arg(m_lastPosition));
1448  return false;
1449  }
1450  }
1452  return true;
1453 }
1454 
1455 /* vim: set expandtab tabstop=4 shiftwidth=4: */
QStringList m_auxFiles
Definition: remotefile.h:98
#define MAX_FILE_CHECK
Definition: remotefile.cpp:37
def write(text, progress=True)
Definition: mythburn.py:279
long long m_readPosition
Definition: remotefile.h:83
static bool DeleteFile(const QString &url)
Definition: remotefile.cpp:417
int restart(void)
Returns milliseconds elapsed since last start() or restart() and resets the count.
Definition: mythtimer.cpp:62
void SetTimeout(bool fast)
A QElapsedTimer based timer to replace use of QTime as a timer.
Definition: mythtimer.h:13
void Reset(void)
Definition: remotefile.cpp:745
static bool MoveFile(const QString &src, const QString &dst, bool overwrite=false)
Definition: remotefile.cpp:676
long long SeekInternal(long long pos, int whence, long long curpos=-1)
Definition: remotefile.cpp:765
static void error(const char *str,...)
Definition: vbi.c:42
QString GetBackendServerIP(void)
Returns the IP address of the locally defined backend IP.
long long m_fileSize
Definition: remotefile.h:81
bool m_completed
Definition: remotefile.h:94
bool m_timeoutIsFast
Definition: remotefile.h:82
static QString FindFile(const QString &filename, const QString &host, const QString &storageGroup, bool useRegex=false, bool allowFallback=false)
Search all BE's for a file in the give storage group.
long long m_lastPosition
Definition: remotefile.h:84
bool IsDataAvailable(void)
Definition: mythsocket.cpp:567
int m_recorderNum
Definition: remotefile.h:86
QStringList m_possibleAuxFiles
Definition: remotefile.h:97
int Write(const char *data, int size)
Definition: mythsocket.cpp:524
static bool CopyFile(const QString &src, const QString &dst, bool overwrite=false, bool verify=false)
Definition: remotefile.cpp:586
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
bool IsThisBackend(const QString &addr)
is this address mapped to this backend host
long long Seek(long long pos, int whence)
Seek to a position within stream; May be unsafe.
bool m_useReadAhead
Definition: remotefile.h:79
QString m_query
Definition: remotefile.h:91
#define O_LARGEFILE
Definition: remotefile.cpp:19
int m_localFile
Definition: remotefile.h:99
void Close(bool haslock=false)
Definition: remotefile.cpp:374
bool IsConnected(void)
Check if both the control and data sockets are currently connected.
bool isRunning(void) const
Returns true if start() or restart() has been called at least once since construction and since any c...
Definition: mythtimer.cpp:134
MythSocket * openSocket(bool control)
Definition: remotefile.cpp:121
bool m_writeMode
Definition: remotefile.h:93
int Read(char *data, int size, int max_wait_ms)
Definition: mythsocket.cpp:537
def read(device=None, features=[])
Definition: disc.py:35
long long GetRealFileSize(void)
GetRealFileSize: returns the current remote file's size.
int GetBackendServerPort(void)
Returns the locally defined backend control port.
bool OpenInternal(void)
Attempts to resume from a disconnected step.
Definition: remotefile.cpp:267
static bool RemoteSendReceiveStringList(const QString &host, QStringList &strlist)
Definition: remotefile.cpp:39
MythSocket * ConnectCommandSocket(const QString &hostname, int port, const QString &announcement, bool *proto_mismatch=nullptr, int maxConnTry=-1, int setup_timeout=-1)
int Write(const void *data, uint count)
Writes data to the end of the write buffer.
bool SendReceiveStringList(QStringList &strlist, bool quickTimeout=false, bool block=true)
Send a message to the backend and wait for a response.
MBASE_PUBLIC QDateTime fromSecsSinceEpoch(uint seconds)
This function takes the number of seconds since the start of the epoch and returns a QDateTime with t...
Definition: mythdate.cpp:88
#define close
Definition: compat.h:16
bool isLocal(void) const
Definition: remotefile.cpp:116
static QString GenMythURL(const QString &host=QString(), int port=0, QString path=QString(), const QString &storageGroup=QString())
QString m_path
Definition: remotefile.h:78
void Flush(void)
Allow DiskLoop() to flush buffer completely ignoring low watermark.
virtual int DecrRef(void)
Decrements reference count and deletes on 0.
int m_timeoutMs
Definition: remotefile.h:80
int Read(void *data, int size)
Definition: remotefile.cpp:935
QString GetMasterHostName(void)
string hostname
Definition: caa.py:17
MythTimer m_lastSizeCheck
Definition: remotefile.h:95
QDateTime LastModified(void) const
RemoteFile(QString url="", bool write=false, bool usereadahead=true, int timeout_ms=2000, const QStringList *possibleAuxiliaryFiles=nullptr)
Definition: remotefile.cpp:72
#define ENO
This can be appended to the LOG args with "+".
Definition: mythlogging.h:99
bool CheckConnection(bool repos=true)
Check current connection and re-establish it if lost.
ThreadedFileWriter * m_fileWriter
Definition: remotefile.h:100
long long GetFileSize(void) const
GetFileSize: returns the remote file's size at the time it was first opened Will query the server in ...
void Reset(void)
Definition: mythsocket.cpp:551
bool Resume(bool repos=true)
Attempts to resume from a disconnected step.
bool ReadStringList(QStringList &list, uint timeoutMS=kShortTimeout)
Definition: mythsocket.cpp:323
static bool Exists(const QString &url, struct stat *fileinfo)
Definition: remotefile.cpp:460
bool SaveAs(QByteArray &data)
bool SetBlocking(bool m_block=true)
Set write blocking mode for the ThreadedFileWriter instance.
bool Open(void)
Opens the file we will be writing to.
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
bool isOpen(void) const
Definition: remotefile.cpp:245
QString FindFile(const QString &filename)
int elapsed(void)
Returns milliseconds elapsed since last start() or restart()
Definition: mythtimer.cpp:90
static QStringList FindFileList(const QString &filename, const QString &host, const QString &storageGroup, bool useRegex=false, bool allowFallback=false)
Search all BE's for files in the give storage group.
bool ReOpen(const QString &newFilename)
Definition: remotefile.cpp:339
bool CheckProtoVersion(MythSocket *socket, uint timeout_ms=kMythSocketLongTimeout, bool error_dialog_desired=false)
bool Open(void)
Definition: remotefile.cpp:254
static const uint kShortTimeout
Definition: mythsocket.h:71
bool IsConnected(void) const
Definition: mythsocket.cpp:561
This class supports the writing of recordings to disk.
MythSocket * m_controlSock
Definition: remotefile.h:89
Class for communcating between myth backends and frontends.
Definition: mythsocket.h:26
QStringList GetFileList(const QString &Path, bool recursive=false)
bool SendReceiveStringList(QStringList &list, uint min_reply_length=0, uint timeoutMS=kLongTimeout)
Definition: mythsocket.cpp:336
long long Seek(long long pos, int whence, long long curpos=-1)
Definition: remotefile.cpp:758
bool WriteStringList(const QStringList &list)
Definition: mythsocket.cpp:311
MythDB * GetMythDB(void)
Definition: mythdb.cpp:46
bool IsMasterBackend(void)
is this the actual MBE process
bool m_canResume
Definition: remotefile.h:85
int Write(const void *data, int size)
Definition: remotefile.cpp:833
QString FileHash(const QString &filename)
QMutex m_lock
Definition: remotefile.h:88
MythSocket * m_sock
Definition: remotefile.h:90
QString GetHostName(void)
void start(void)
starts measuring elapsed time.
Definition: mythtimer.cpp:47
static QString GetFileHash(const QString &url)
Definition: remotefile.cpp:552
bool SetBlocking(bool block=true)
Set write blocking mode While in blocking mode, ThreadedFileWriter::Write will wait for buffers to be...