libpappsomspp
Library for mass spectrometry
timsbindec.cpp
Go to the documentation of this file.
1 /**
2  * \file pappsomspp/vendors/tims/timsbindec.h
3  * \date 23/08/2019
4  * \author Olivier Langella
5  * \brief binary file handler of Bruker's TimsTof raw data
6  */
7 
8 /*******************************************************************************
9  * Copyright (c) 2019 Olivier Langella <Olivier.Langella@u-psud.fr>.
10  *
11  * This file is part of the PAPPSOms++ library.
12  *
13  * PAPPSOms++ is free software: you can redistribute it and/or modify
14  * it under the terms of the GNU General Public License as published by
15  * the Free Software Foundation, either version 3 of the License, or
16  * (at your option) any later version.
17  *
18  * PAPPSOms++ is distributed in the hope that it will be useful,
19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21  * GNU General Public License for more details.
22  *
23  * You should have received a copy of the GNU General Public License
24  * along with PAPPSOms++. If not, see <http://www.gnu.org/licenses/>.
25  *
26  ******************************************************************************/
27 
28 #include "timsbindec.h"
29 #include "../../../pappsomspp/pappsoexception.h"
30 #include "../../../pappsomspp/exception/exceptionnotimplemented.h"
31 #include "timsframetype1.h"
32 #include <QDataStream>
33 #include <QThread>
34 #include <zstd.h>
35 #include "timsframerawdatachunck.h"
36 #include "timsdata.h"
37 
38 using namespace pappso;
39 
40 TimsBinDec::TimsBinDec(const QFileInfo &timsBinFile, int timsCompressionType)
41  : m_timsBinFile(timsBinFile.absoluteFilePath())
42 {
43  m_timsCompressionType = timsCompressionType;
44  if((timsCompressionType != 1) && (timsCompressionType != 2))
45  {
47  QObject::tr("compression type %1 not handled by this library")
48  .arg(timsCompressionType));
49  }
50  if(m_timsBinFile.isEmpty())
51  {
53  QObject::tr("No TIMS binary file name specified"));
54  }
55  QFile file(m_timsBinFile);
56  if(!file.open(QIODevice::ReadOnly))
57  {
58  throw PappsoException(
59  QObject::tr("ERROR opening TIMS binary file %1 for read")
60  .arg(timsBinFile.absoluteFilePath()));
61  }
62 }
63 
64 
66  : m_timsBinFile(other.m_timsBinFile)
67 {
68 }
69 
71 {
72 
73 
74  if(mpa_decompressMemoryBuffer != nullptr)
75  {
77  }
78 
79  if(mp_fileLinear != nullptr)
80  {
81  mp_fileLinear->close();
82  delete mp_fileLinear;
83  }
84  if(mp_fileRandom != nullptr)
85  {
86  mp_fileRandom->close();
87  delete mp_fileRandom;
88  }
89 }
90 
91 void
93 {
94  qDebug();
95  if(mp_fileLinear != nullptr)
96  {
97  mp_fileLinear->close();
98  delete mp_fileLinear;
99  }
100  mp_fileLinear = nullptr;
101  m_firstFrameId = 0;
102  m_lastFrameId = 0;
103  m_linearAccessRawDataChunckList.resize(0);
104  qDebug();
105 }
106 
107 
108 QFile *
110  std::size_t frameId,
111  const std::vector<pappso::TimsFrameRecord> &frame_record_list)
112 {
113  if(mp_fileLinear == nullptr)
114  {
115  mp_fileLinear = new QFile(m_timsBinFile);
116  if(!mp_fileLinear->open(QIODevice::ReadOnly))
117  {
118  throw PappsoException(
119  QObject::tr("ERROR opening TIMS binary file %1 for read")
120  .arg(m_timsBinFile));
121  }
122 
124  frameId, m_linearAccessRawDataChunckDequeSize, frame_record_list);
125  }
126 
127  return mp_fileLinear;
128 }
129 
130 QFile *
132 {
133  if(mp_fileRandom == nullptr)
134  {
135  mp_fileRandom = new QFile(m_timsBinFile);
136  if(!mp_fileRandom->open(QIODevice::ReadOnly))
137  {
138  throw PappsoException(
139  QObject::tr("ERROR opening TIMS binary file %1 for read")
140  .arg(m_timsBinFile));
141  }
142  }
143  return mp_fileRandom;
144 }
145 
148  std::size_t frameId,
149  const std::vector<pappso::TimsFrameRecord> &frame_record_list)
150 {
151 
152  qDebug() << "frameId:" << frameId;
153 
154  // QMutexLocker locker(&m_mutex);
155  QFile *p_file = getQfileLinear(frameId, frame_record_list);
156 
157  if(mp_fileLinear->pos() <
158  (qint64)frame_record_list[m_firstFrameId].tims_offset)
159  {
160  }
161  else
162  {
163  if(frameId > m_lastFrameId)
164  {
166  {
167  // move forward
168  moveLinearReadForward(frame_record_list);
169  }
170  }
171  }
172 
173  auto it = std::find_if(m_linearAccessRawDataChunckList.begin(),
175  [frameId](const TimsFrameRawDataChunck &chunck) {
176  if(chunck.getFrameId() == frameId)
177  return true;
178  return false;
179  });
180  if(it != m_linearAccessRawDataChunckList.end())
181  {
182  try
183  {
184  return getTimsFrameFromRawDataChunck(*it);
185  }
186  catch(PappsoException &error)
187  {
188 
189  throw PappsoException(
190  QObject::tr("ERROR reading TIMS binary file %1 with linear QFile: "
191  "%2")
192  .arg(m_timsBinFile)
193  .arg(error.qwhat()));
194  }
195  }
196 
197  // random access file
198  qDebug();
199  p_file = getQfileRandom();
200  bool seekpos_ok = p_file->seek(frame_record_list[frameId].tims_offset);
201  if(!seekpos_ok)
202  {
203  throw PappsoException(
204  QObject::tr("ERROR reading TIMS frame %1 TIMS binary file %2: "
205  "m_timsBinFile.seek(%3) failed")
206  .arg(frameId)
207  .arg(m_timsBinFile)
208  .arg(frame_record_list[frameId].tims_offset));
209  }
210 
211 
212  try
213  {
215  p_file, frameId, frame_record_list);
216  }
217  catch(PappsoException &error)
218  {
219 
220  throw PappsoException(
221  QObject::tr("ERROR reading TIMS binary file %1 with random QFile: "
222  "%2")
223  .arg(m_timsBinFile)
224  .arg(error.qwhat()));
225  }
226 
228 }
229 
230 void
232  const std::vector<pappso::TimsFrameRecord> &frame_record_list)
233 {
234  qDebug();
235  for(std::size_t i = 0; i < m_linearForwardThreshold; i++)
236  {
237  auto it = std::min_element(
238  m_linearAccessRawDataChunckList.begin(),
239  m_linearAccessRawDataChunckList.end(),
240  [](const TimsFrameRawDataChunck &a, const TimsFrameRawDataChunck &b) {
241  return a.getFrameId() < b.getFrameId();
242  });
243  m_lastFrameId++;
244  m_firstFrameId++;
245  if(m_lastFrameId >= frame_record_list.size())
246  break;
247  it->readTimsFrame(mp_fileLinear, m_lastFrameId, frame_record_list);
248  }
249  qDebug();
250 }
251 
252 
255  const TimsFrameRawDataChunck &raw_data_chunck)
256 {
257  qDebug();
258  TimsFrameSPtr frame_sptr;
259  if(raw_data_chunck.getCompressedSize() > 0)
260  {
261  qDebug();
262  if(m_timsCompressionType == 2)
263  {
264  auto decompressed_size2 =
265  ZSTD_getFrameContentSize(raw_data_chunck.getMemoryBuffer(),
266  raw_data_chunck.getCompressedSize());
267  qDebug();
268  if(decompressed_size2 == ZSTD_CONTENTSIZE_UNKNOWN)
269  {
270  throw PappsoException(
271  QObject::tr("ERROR TimsBinDec::getTimsFrameFromRawDataChunck "
272  "reading TIMS frame %1 TIMS binary file %2: "
273  " decompressed_size2 == ZSTD_CONTENTSIZE_UNKNOWN, "
274  "frame_length=%3")
275  .arg(raw_data_chunck.getFrameId())
276  .arg(m_timsBinFile)
277  .arg(raw_data_chunck.getFrameLength()));
278  }
279  qDebug();
280  if(decompressed_size2 == ZSTD_CONTENTSIZE_ERROR)
281  {
282  qDebug();
283  throw PappsoException(
284  QObject::tr("ERROR TimsBinDec::getTimsFrameFromRawDataChunck "
285  "reading TIMS frame %1 TIMS binary file %2: "
286  " decompressed_size2 == ZSTD_CONTENTSIZE_ERROR, "
287  "frame_length=%3")
288  .arg(raw_data_chunck.getFrameId())
289  .arg(m_timsBinFile)
290  .arg(raw_data_chunck.getFrameLength()));
291  }
292  qDebug() << " decompressed_size2=" << decompressed_size2;
293 
294  if(m_decompressMemoryBufferSize < (decompressed_size2 + 10))
295  {
296  if(mpa_decompressMemoryBuffer != nullptr)
297  {
299  }
300  m_decompressMemoryBufferSize = decompressed_size2 + 10;
303  }
304  std::size_t decompressed_size =
305  ZSTD_decompress(mpa_decompressMemoryBuffer,
307  raw_data_chunck.getMemoryBuffer(),
308  raw_data_chunck.getCompressedSize());
309  qDebug();
310 
311  if(decompressed_size != decompressed_size2)
312  {
313  throw PappsoException(
314  QObject::tr("ERROR TimsBinDec::getTimsFrameFromRawDataChunck "
315  "reading TIMS frame %1 TIMS binary file %2: "
316  "decompressed_size != decompressed_size2")
317  .arg(raw_data_chunck.getFrameId())
318  .arg(m_timsBinFile)
319  .arg(decompressed_size)
320  .arg(decompressed_size2));
321  }
322 
323  qDebug();
324 
325  frame_sptr =
326  std::make_shared<TimsFrame>(raw_data_chunck.getFrameId(),
327  raw_data_chunck.getFrameNumberOfScans(),
329  decompressed_size);
330  }
331  else
332  {
333 
334  if(m_timsCompressionType == 1)
335  {
336  frame_sptr = std::make_shared<TimsFrameType1>(
337  raw_data_chunck.getFrameId(),
338  raw_data_chunck.getFrameNumberOfScans(),
339  raw_data_chunck.getMemoryBuffer(),
340  raw_data_chunck.getCompressedSize());
341  }
342  }
343  // delete[] mpa_decompressMemoryBuffer;
344  }
345  else
346  {
347  frame_sptr =
348  std::make_shared<TimsFrame>(raw_data_chunck.getFrameId(),
349  raw_data_chunck.getFrameNumberOfScans(),
350  nullptr,
351  0);
352  }
353  return frame_sptr;
354 }
355 /*
356 TimsFrameCstSPtr
357 TimsBinDec::getTimsFrameCstSPtr(std::size_t timsId)
358 {
359  return getTimsFrameCstSPtrByOffset(timsId, m_indexArray[timsId]);
360 }
361 */
362 
363 void
365  std::size_t start_frame_id,
366  std::size_t chunk_deque_size,
367  const std::vector<pappso::TimsFrameRecord> &frame_record_list)
368 {
369  qDebug();
370  m_linearAccessRawDataChunckList.resize(chunk_deque_size);
371  m_firstFrameId = start_frame_id;
372  m_lastFrameId = start_frame_id;
373 
374  QFile *p_file = mp_fileLinear;
375  if(p_file == nullptr)
376  {
377  throw PappsoException(QObject::tr("ERROR mp_fileLinear == nullptr"));
378  }
379 
380  bool seekpos_ok = p_file->seek(frame_record_list[start_frame_id].tims_offset);
381  if(!seekpos_ok)
382  {
383  throw PappsoException(
384  QObject::tr("ERROR reading TIMS frame %1 TIMS binary file %2: "
385  "m_timsBinFile.seek(%3) failed")
386  .arg(start_frame_id)
387  .arg(m_timsBinFile)
388  .arg(frame_record_list[start_frame_id].tims_offset));
389  }
390 
391  try
392  {
393  for(TimsFrameRawDataChunck &chunck : m_linearAccessRawDataChunckList)
394  {
395 
396  chunck.readTimsFrame(p_file, start_frame_id, frame_record_list);
397  m_lastFrameId = start_frame_id;
398  start_frame_id++;
399  }
400  }
401  catch(PappsoException &error)
402  {
403 
404  throw PappsoException(
405  QObject::tr(
406  "ERROR in TimsBinDec::startLinearRead reading TIMS binary file %1:\n "
407  " start_frame_id=%2 m_firstFrameId=%3 m_lastFrameId=%4 "
408  "%5")
409  .arg(m_timsBinFile)
410  .arg(start_frame_id)
411  .arg(m_firstFrameId)
412  .arg(m_lastFrameId)
413  .arg(error.qwhat()));
414  }
415 
416  qDebug();
417 }
virtual const QString & qwhat() const
QFile * mp_fileRandom
Definition: timsbindec.h:103
std::size_t m_firstFrameId
Definition: timsbindec.h:113
std::size_t m_decompressMemoryBufferSize
Definition: timsbindec.h:107
TimsFrameSPtr getTimsFrameSPtrByOffset(std::size_t frameId, const std::vector< pappso::TimsFrameRecord > &frame_record_list)
Definition: timsbindec.cpp:147
void startLinearRead(std::size_t start_frame_id, std::size_t chunk_deque_size, const std::vector< pappso::TimsFrameRecord > &frame_record_list)
populate a fifo buffer with TimsFrameRawDataChunck accelerates inputs from file
Definition: timsbindec.cpp:364
QFile * getQfileRandom()
open one QFile handler for random read
Definition: timsbindec.cpp:131
std::size_t m_linearForwardThreshold
Definition: timsbindec.h:116
void closeLinearRead()
close file access and flush cache
Definition: timsbindec.cpp:92
std::size_t m_lastFrameId
Definition: timsbindec.h:114
std::size_t m_linearAccessRawDataChunckDequeSize
Definition: timsbindec.h:115
QFile * getQfileLinear(std::size_t frameId, const std::vector< pappso::TimsFrameRecord > &frame_record_list)
open one QFile handler for linear read
Definition: timsbindec.cpp:109
QFile * mp_fileLinear
Definition: timsbindec.h:102
std::vector< TimsFrameRawDataChunck > m_linearAccessRawDataChunckList
Definition: timsbindec.h:112
void moveLinearReadForward(const std::vector< pappso::TimsFrameRecord > &frame_record_list)
Definition: timsbindec.cpp:231
TimsBinDec(const QFileInfo &timsBinFile, int timsCompressionType)
Definition: timsbindec.cpp:40
virtual ~TimsBinDec()
Definition: timsbindec.cpp:70
TimsFrameSPtr getTimsFrameFromRawDataChunck(const TimsFrameRawDataChunck &raw_data_chunck)
Definition: timsbindec.cpp:254
QString m_timsBinFile
Definition: timsbindec.h:101
TimsFrameRawDataChunck m_randemAccessFrameRawDataChunck
Definition: timsbindec.h:109
char * mpa_decompressMemoryBuffer
Definition: timsbindec.h:106
bool readTimsFrame(QFile *p_file, std::size_t frameId, const std::vector< pappso::TimsFrameRecord > &frame_record_list)
tries to keep as much as possible monoisotopes, removing any possible C13 peaks and changes multichar...
Definition: aa.cpp:39
std::shared_ptr< TimsFrame > TimsFrameSPtr
Definition: timsframe.h:40
binary file handler of Bruker's TimsTof raw data
main Tims data handler
stores raw binary tims frame
handle a single Bruker's TimsTof frame type 1 compression