MythTV  master
mythhttpencoding.cpp
Go to the documentation of this file.
1 // MythTV
2 #include "mythlogging.h"
3 #include "unziputil.h"
5 #include "http/mythhttpdata.h"
6 #include "http/mythhttpfile.h"
9 
10 // Qt
11 #include <QDomDocument>
12 #include <QJsonDocument>
13 #include <QJsonObject>
14 #include <QJsonValue>
15 
16 #define LOC QString("HTTPEnc: ")
17 
29 using MimePair = std::pair<float,QString>;
30 QStringList MythHTTPEncoding::GetMimeTypes(const QString &Accept)
31 {
32  // Split out mime types
33 #if QT_VERSION < QT_VERSION_CHECK(5,14,0)
34  auto types = Accept.split(",", QString::SkipEmptyParts);
35 #else
36  auto types = Accept.split(",", Qt::SkipEmptyParts);
37 #endif
38 
39  std::vector<MimePair> weightings;
40  for (const auto & type : types)
41  {
42  QString mime = type.trimmed();
43  auto quality = 1.0F;
44  // Find any quality value (defaults to 1)
45  if (auto index = type.lastIndexOf(";"); index > -1)
46  {
47  mime = type.mid(0, index).trimmed().toLower();
48  auto qual = type.mid(index + 1).trimmed();
49  if (auto index2 = qual.lastIndexOf("="); index2 > -1)
50  {
51  bool ok = false;
52  auto newquality = qual.mid(index2 + 1).toFloat(&ok);
53  if (ok)
54  quality = newquality;
55  }
56  }
57  weightings.emplace_back(quality, mime);
58  }
59 
60  // Sort the list
61  auto comp = [](const MimePair& First, const MimePair& Second) { return First.first > Second.first; };
62  std::sort(weightings.begin(), weightings.end(), comp);
63 
64  // Build the final result. This will pass through invalid types - which should
65  // be handled by the consumer (e.g. wildcard specifiers are not handled).
66  QStringList result;
67  for (const auto & weight : weightings)
68  result.append(weight.second);
69 
70  // Default to xml
71  if (result.empty())
72  result.append("application/xml");
73  return result;
74 }
75 
79 {
80  if (!Request || !Request->m_content.get())
81  return;
82 
83  auto contenttype = MythHTTP::GetHeader(Request->m_headers, "content-type");
84 
85  // type is e.g. text/html; charset=UTF-8 or multipart/form-data; boundary=something
86 #if QT_VERSION < QT_VERSION_CHECK(5,14,0)
87  auto types = contenttype.split(";", QString::SkipEmptyParts);
88 #else
89  auto types = contenttype.split(";", Qt::SkipEmptyParts);
90 #endif
91 
92  if (types.isEmpty())
93  return;
94 
95  // Note: This can produce an invalid mime type but there is no sensible fallback
96  if (auto mime = MythMimeDatabase::MimeTypeForName(types[0].trimmed().toLower()); mime.IsValid())
97  {
98  Request->m_content->m_mimeType = mime;
99  if (mime.Name() == "application/x-www-form-urlencoded")
101  else if (mime.Name() == "text/xml" || mime.Name() == "application/xml" ||
102  mime.Name() == "application/soap+xml")
104  else if (mime.Name() == "application/json")
106  else
107  LOG(VB_HTTP, LOG_ERR, QString("Don't know how to get the parameters for MIME type: '%1'").arg(mime.Name()));
108  }
109  else
110  LOG(VB_HTTP, LOG_ERR, QString("Unknown MIME type: '%1'").arg(types[0]));
111 
112 }
113 
115 {
116  if (!Request || !Request->m_content.get())
117  return;
118 
119  auto payload = QString::fromUtf8(Request->m_content->constData(), Request->m_content->size());
120 
121  // This looks odd, but it is here to cope with stupid UPnP clients that
122  // forget to de-escape the URLs. We can't map %26 here as well, as that
123  // breaks anything that is trying to pass & as part of a name or value.
124  payload.replace("&amp;", "&");
125  if (!payload.isEmpty())
126  {
127 #if QT_VERSION < QT_VERSION_CHECK(5,14,0)
128  QStringList params = payload.split('&', QString::SkipEmptyParts);
129 #else
130  QStringList params = payload.split('&', Qt::SkipEmptyParts);
131 #endif
132  for (const auto & param : qAsConst(params))
133  {
134  QString name = param.section('=', 0, 0);
135  QString value = param.section('=', 1);
136  value.replace("+", " ");
137  if (!name.isEmpty())
138  {
139  name = QUrl::fromPercentEncoding(name.toUtf8());
140  value = QUrl::fromPercentEncoding(value.toUtf8());
141  Request->m_queries.insert(name.trimmed().toLower(), value);
142  }
143  }
144  }
145 }
146 
148 {
149  LOG(VB_HTTP, LOG_DEBUG, "Inspecting XML payload");
150 
151  if (!Request || !Request->m_content.get())
152  return;
153 
154  // soapaction is formatted like "\"http://mythtv.org/Dvr/GetRecordedList\""
155  QString soapaction = Request->m_headers->value("soapaction");
156  soapaction.remove('"');
157  int lastSlashPos= soapaction.lastIndexOf('/');
158  if (lastSlashPos < 0)
159  return;
160  if (Request->m_path == "/")
161  Request->m_path.append(Request->m_fileName).append("/");
162  Request->m_fileName = soapaction.right(soapaction.size()-lastSlashPos-1);
163  LOG(VB_HTTP, LOG_DEBUG, QString("Found method call (%1)").arg(Request->m_fileName));
164 
165  auto payload = QDomDocument();
166  QString err_msg;
167  int err_line {-1};
168  int err_col {-1};
169  if (!payload.setContent(static_cast<QByteArray>(Request->m_content->constData()),
170  true, &err_msg, &err_line, &err_col))
171  {
172  LOG(VB_HTTP, LOG_WARNING, "Unable to parse XML request body");
173  LOG(VB_HTTP, LOG_WARNING, QString("- Error at line %1, column %2, msg: %3")
174  .arg(err_line).arg(err_col).arg(err_msg));
175  return;
176  }
177  QString doc_name = payload.documentElement().localName();
178  if (doc_name.compare("envelope", Qt::CaseInsensitive) == 0)
179  {
180  LOG(VB_HTTP, LOG_DEBUG, "Found SOAP XML message envelope");
181  auto doc_body = payload.documentElement().namedItem("Body");
182  if (doc_body.isNull() || !doc_body.hasChildNodes()) // None or empty body
183  {
184  LOG(VB_HTTP, LOG_DEBUG, "Missing or empty SOAP body");
185  return;
186  }
187  auto body_contents = doc_body.firstChild();
188  if (body_contents.hasChildNodes()) // params for the method
189  {
190  for (QDomNode node = body_contents.firstChild(); !node.isNull(); node = node.nextSibling())
191  {
192  QString name = node.localName();
193  QString value = node.toElement().text();
194  if (!name.isEmpty())
195  {
196  // TODO: html decode entities if required
197  Request->m_queries.insert(name.trimmed().toLower(), value);
198  LOG(VB_HTTP, LOG_DEBUG, QString("Found URL param (%1=%2)").arg(name, value));
199  }
200  }
201  }
202  }
203 }
204 
206 {
207  LOG(VB_HTTP, LOG_DEBUG, "Inspecting JSON payload");
208 
209  if (!Request || !Request->m_content.get())
210  return;
211 
212  QByteArray jstr = static_cast<QByteArray>(Request->m_content->constData());
213  QJsonParseError parseError {};
214  QJsonDocument doc = QJsonDocument::fromJson(jstr, &parseError);
215  if (parseError.error != QJsonParseError::NoError)
216  {
217  LOG(VB_HTTP, LOG_WARNING,
218  QString("Unable to parse JSON request body - Error at position %1, msg: %2")
219  .arg(parseError.offset).arg(parseError.errorString()));
220  return;
221  }
222 
223  QJsonObject json = doc.object();
224  foreach(const QString& key, json.keys())
225  {
226  if (!key.isEmpty())
227  {
228  QString value;
229  if (json.value(key).isObject())
230  {
231  QJsonDocument vd(json.value(key).toObject());
232  value = vd.toJson(QJsonDocument::Compact);
233  }
234  else
235  {
236  value = json.value(key).toVariant().toString();
237 
238  if (value.isEmpty())
239  {
240  LOG(VB_HTTP, LOG_WARNING,
241  QString("Failed to parse value for key '%1' from %2")
242  .arg(key, QString(jstr)));
243  }
244  }
245 
246  Request->m_queries.insert(key.trimmed().toLower(), value);
247  LOG(VB_HTTP, LOG_DEBUG,
248  QString("Found URL param (%1=%2)").arg(key, value));
249  }
250  }
251 }
252 
256 {
257  auto * data = std::get_if<HTTPData>(&Content);
258  auto * file = std::get_if<HTTPFile>(&Content);
259  if (!(data || file))
260  return {};
261 
262  QString filename = data ? (*data)->m_fileName : file ? (*file)->m_fileName : "";
263 
264  // Look for unambiguous mime type
266  if (types.size() == 1)
267  return types.front();
268 
269  // Look for an override. QMimeDatabase gets it wrong sometimes when the result
270  // is ambiguous and it resorts to probing. Add to this list as necessary
271  static const std::map<QString,QString> s_mimeOverrides =
272  {
273  { "ts", "video/mp2t"}
274  };
275 
277  for (const auto & type : s_mimeOverrides)
278  if (suffix.compare(type.first, Qt::CaseInsensitive) == 0)
280 
281  // Try interrogating content as well
282  if (data)
283  if (auto mime = MythMimeDatabase::MimeTypeForFileNameAndData(filename, **data); mime.IsValid())
284  return mime;
285  if (file)
286  if (auto mime = MythMimeDatabase::MimeTypeForFileNameAndData(filename, (*file).get()); mime.IsValid())
287  return mime;
288 
289  // Default to text/plain (possibly use application/octet-stream as well?)
290  return MythMimeDatabase::MimeTypeForName("text/plain");
291 }
292 
301 {
302  auto result = HTTPNoEncode;
303  if (!Response || !Response->m_requestHeaders)
304  return result;
305 
306  // We need something to compress/chunk
307  auto * data = std::get_if<HTTPData>(&Response->m_response);
308  auto * file = std::get_if<HTTPFile>(&Response->m_response);
309  if (!(data || file))
310  return result;
311 
312  // Don't touch range requests. They do not work with compression and there
313  // is no point in chunking gzipped content as the client still has to wait
314  // for the entire payload before unzipping
315  // Note: It is permissible to chunk a range request - but ignore for the
316  // timebeing to keep the code simple.
317  if ((data && !(*data)->m_ranges.empty()) || (file && !(*file)->m_ranges.empty()))
318  return result;
319 
320  // Has the client actually requested compression
321  bool wantgzip = MythHTTP::GetHeader(Response->m_requestHeaders, "accept-encoding").toLower().contains("gzip");
322 
323  // Chunking is HTTP/1.1 only - and must be supported
324  bool chunkable = Response->m_version == HTTPOneDotOne;
325 
326  // Has the client requested no chunking by specifying identity?
327  bool allowchunk = ! MythHTTP::GetHeader(Response->m_requestHeaders, "accept-encoding").toLower().contains("identity");
328 
329  // and restrict to 'chunky' files
330  bool chunky = Size > 102400; // 100KB
331 
332  // Don't compress anything that is too large. Under normal circumstances this
333  // should not be a problem as we only compress text based data - but avoid
334  // potentially memory hungry compression operations.
335  // On the flip side, don't compress trivial amounts of data
336  bool gzipsize = Size > 512 && !chunky; // 0.5KB <-> 100KB
337 
338  // Only consider compressing text based content. No point in compressing audio,
339  // video and images.
340  bool compressable = (data ? (*data)->m_mimeType : (*file)->m_mimeType).Inherits("text/plain");
341 
342  // Decision time
343  bool gzip = wantgzip && gzipsize && compressable;
344  bool chunk = chunkable && chunky && allowchunk;
345 
346  if (!gzip)
347  {
348  // Chunking happens as we write to the socket, so flag it as required
349  if (chunk)
350  {
351  result = HTTPChunked;
352  if (data) (*data)->m_encoding = result;
353  if (file) (*file)->m_encoding = result;
354  }
355  return result;
356  }
357 
358  // As far as I can tell, Qt's implicit sharing of data should ensure we aren't
359  // copying data unnecessarily here - but I can't be sure. We could definitely
360  // improve compressing files by avoiding the copy into a temporary buffer.
361  HTTPData buffer = MythHTTPData::Create(data ? gzipCompress(**data) : gzipCompress((*file)->readAll()));
362 
363  // Add the required header
364  Response->AddHeader("Content-Encoding", "gzip");
365 
366  LOG(VB_HTTP, LOG_INFO, LOC + QString("'%1' compressed from %2 to %3 bytes")
367  .arg(data ? (*data)->m_fileName : (*file)->fileName())
368  .arg(Size).arg(buffer->size()));
369 
370  // Copy the filename and last modified, set the new buffer and set the content size
371  buffer->m_lastModified = data ? (*data)->m_lastModified : (*file)->m_lastModified;
372  buffer->m_etag = data ? (*data)->m_etag : (*file)->m_etag;
373  buffer->m_fileName = data ? (*data)->m_fileName : (*file)->m_fileName;
374  buffer->m_cacheType = data ? (*data)->m_cacheType : (*file)->m_cacheType;
375  buffer->m_encoding = HTTPGzip;
376  Response->m_response = buffer;
377  Size = buffer->size();
378  return HTTPGzip;
379 }
HTTPChunked
@ HTTPChunked
Definition: mythhttptypes.h:137
MythHTTPRequest
Limited parsing of HTTP method and some headers to determine validity of request.
Definition: mythhttprequest.h:12
unziputil.h
HTTPVariant
std::variant< std::monostate, HTTPData, HTTPFile > HTTPVariant
Definition: mythhttptypes.h:41
MythHTTP::GetHeader
static QString GetHeader(const HTTPHeaders &Headers, const QString &Value, const QString &Default="")
Definition: mythhttptypes.h:316
MythMimeDatabase::MimeTypesForFileName
static MythMimeTypes MimeTypesForFileName(const QString &FileName)
Return a vector of mime types that match the given filename.
Definition: mythmimedatabase.cpp:175
MythHTTPResponse
Definition: mythhttpresponse.h:16
types
static const struct wl_interface * types[]
Definition: idle_inhibit_unstable_v1.c:39
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:39
build_compdb.file
file
Definition: build_compdb.py:55
MythHTTPEncoding::GetContentType
static void GetContentType(MythHTTPRequest *Request)
Parse the incoming Content-Type header for POST/PUT content.
Definition: mythhttpencoding.cpp:78
MythHTTPResponse::m_version
MythHTTPVersion m_version
Definition: mythhttpresponse.h:53
mythhttpdata.h
HTTPData
std::shared_ptr< MythHTTPData > HTTPData
Definition: mythhttptypes.h:36
mythhttpfile.h
MythHTTPEncode
MythHTTPEncode
Definition: mythhttptypes.h:133
mythlogging.h
MythHTTPEncoding::GetMimeType
static MythMimeType GetMimeType(HTTPVariant Content)
Return a QMimeType that represents Content.
Definition: mythhttpencoding.cpp:255
MythHTTPData::Create
static HTTPData Create()
Definition: mythhttpdata.cpp:4
MythHTTPResponse::m_response
HTTPVariant m_response
Definition: mythhttpresponse.h:61
MythHTTPEncoding::GetURLEncodedParameters
static void GetURLEncodedParameters(MythHTTPRequest *Request)
Definition: mythhttpencoding.cpp:114
gzipCompress
QByteArray gzipCompress(const QByteArray &data)
Definition: unziputil.cpp:93
mythhttpencoding.h
MythMimeDatabase::SuffixForFileName
static QString SuffixForFileName(const QString &FileName)
Return the preferred suffix for the given filename.
Definition: mythmimedatabase.cpp:187
mythhttpresponse.h
MimePair
std::pair< float, QString > MimePair
Parse the incoming HTTP 'Accept' header and return an ordered list of preferences.
Definition: mythhttpencoding.cpp:29
mythmimedatabase.h
LOC
#define LOC
Definition: mythhttpencoding.cpp:16
MythMimeDatabase::MimeTypeForName
static MythMimeType MimeTypeForName(const QString &Name)
Return a mime type that matches the given name.
Definition: mythmimedatabase.cpp:201
HTTPOneDotOne
@ HTTPOneDotOne
Definition: mythhttptypes.h:85
hardwareprofile.distros.mythtv_data.request.Request
def Request(url=None)
Definition: distros/mythtv_data/request.py:64
MythHTTPEncoding::GetJSONEncodedParameters
static void GetJSONEncodedParameters(MythHTTPRequest *Request)
Definition: mythhttpencoding.cpp:205
MythHTTPEncoding::GetXMLEncodedParameters
static void GetXMLEncodedParameters(MythHTTPRequest *Request)
Definition: mythhttpencoding.cpp:147
MythMimeType::IsValid
bool IsValid() const
Definition: mythmimetype.cpp:26
MythMimeType
Definition: mythmimetype.h:16
HTTPNoEncode
@ HTTPNoEncode
Definition: mythhttptypes.h:135
MythHTTPEncoding::Compress
static MythHTTPEncode Compress(MythHTTPResponse *Response, int64_t &Size)
Compress the response content under certain circumstances or mark the content as 'chunkable'.
Definition: mythhttpencoding.cpp:300
MythHTTPResponse::m_requestHeaders
HTTPHeaders m_requestHeaders
Definition: mythhttpresponse.h:59
HTTPGzip
@ HTTPGzip
Definition: mythhttptypes.h:136
MythHTTPResponse::AddHeader
std::enable_if< std::is_convertible< T, QString >::value, void >::type AddHeader(const QString &key, const T &val)
Definition: mythhttpresponse.h:39
MythHTTPEncoding::GetMimeTypes
static QStringList GetMimeTypes(const QString &Accept)
Definition: mythhttpencoding.cpp:30
build_compdb.filename
filename
Definition: build_compdb.py:21
MythMimeDatabase::MimeTypeForFileNameAndData
static MythMimeType MimeTypeForFileNameAndData(const QString &FileName, const QByteArray &Data)
Return a mime type for the given FileName and data.
Definition: mythmimedatabase.cpp:217
Content
Definition: content.h:36