UltraScan III
us_dataIO.cpp
Go to the documentation of this file.
1 #include <QDomDocument>
3 
4 #include "us_dataIO.h"
5 #include "us_crc.h"
6 #include "us_math2.h"
7 #include "us_matrix.h"
8 #include "us_util.h"
9 
10 // Return the count of readings points
12 {
13  return xvalues.size();
14 }
15 
16 // Return the count of scans
18 {
19  return scanData.size();
20 }
21 
22 // Return the index of an X (radius or wavelength) value
23 int US_DataIO::RawData::xindex( double xvalue )
24 {
25  return index( xvalues, xvalue );
26 }
27 
28 // Get the X value (radius) at a given index
29 double US_DataIO::RawData::radius( int radx )
30 {
31  return xvalues[ radx ];
32 }
33 
34 // Get the X value (wavelength) at a given index
36 {
37  return xvalues[ wavx ];
38 }
39 
40 // Get the readings value at given scan,radius indecies
41 double US_DataIO::RawData::value( int scnx, int radx )
42 {
43  return scanData[ scnx ].rvalues[ radx ];
44 }
45 // Get the readings value at given scan,radius indecies
46 double US_DataIO::RawData::reading( int scnx, int radx )
47 {
48  return scanData[ scnx ].rvalues[ radx ];
49 }
50 
51 // Set the readings value at given scan,radius indecies
52 bool US_DataIO::RawData::setValue( int scnx, int radx, double value )
53 {
54  if ( scnx < 0 || scnx >= scanData.size() ||
55  radx < 0 || radx >= scanData[ scnx ].rvalues.size() )
56  return false;
57 
58  scanData[ scnx ].rvalues[ radx ] = value;
59  return true;
60 }
61 
62 // Get the standard deviation value at given scan,radius indecies
63 double US_DataIO::RawData::std_dev( int scnx, int radx )
64 {
65  US_DataIO::Scan* scn = &scanData[ scnx ];
66  bool use_stddev = ( scn->nz_stddev && scn->stddevs.size() > radx );
67  return ( use_stddev ? scn->stddevs[ radx ] : 0.0 );
68 }
69 
70 // Calculate the average temperature value across scans
72 {
73  int scansize = scanData.size();
74  double sum = 0.0;
75 
76  for ( int ii = 0; ii < scansize; ii++ )
77  {
78  sum += scanData[ ii ].temperature;
79  }
80 
81  return ( sum / (double)scansize );
82 }
83 
84 // Calculate the spread of temperature values across scans
86 {
87  int scansize = scanData.size();
88  double smin = 99.0;
89  double smax = 0.0;
90 
91  for ( int ii = 0; ii < scansize; ii++ )
92  {
93  double temp = scanData[ ii ].temperature;
94  smin = qMin( smin, temp );
95  smax = qMax( smax, temp );
96  }
97 
98  return qAbs( smax - smin );
99 }
100 
101 // Return the count of readings points
103 {
104  return xvalues.size();
105 }
106 
107 // Return the count of scans
109 {
110  return scanData.size();
111 }
112 
113 // Return the index of an X (radius or wavelength) value
114 int US_DataIO::EditedData::xindex( double xvalue )
115 {
116  return index( xvalues, xvalue );
117 }
118 
119 // Get the X value (radius) at a given index
121 {
122  return xvalues[ radx ];
123 }
124 
125 // Get the X value (wavelength) at a given index
127 {
128  return xvalues[ wavx ];
129 }
130 
131 // Get the readings value at given scan,radius indecies
132 double US_DataIO::EditedData::value( int scnx, int radx )
133 {
134  return scanData[ scnx ].rvalues[ radx ];
135 }
136 
137 // Get the readings value at given scan,radius indecies
138 double US_DataIO::EditedData::reading( int scnx, int radx )
139 {
140  return scanData[ scnx ].rvalues[ radx ];
141 }
142 
143 // Set the readings value at given scan,radius indecies
144 bool US_DataIO::EditedData::setValue( int scnx, int radx, double value )
145 {
146  if ( scnx < 0 || scnx >= scanData.size() ||
147  radx < 0 || radx >= scanData[ scnx ].rvalues.size() )
148  return false;
149 
150  scanData[ scnx ].rvalues[ radx ] = value;
151  return true;
152 }
153 
154 // Get the standard deviation value at given scan,radius indecies
155 double US_DataIO::EditedData::std_dev( int scnx, int radx )
156 {
157  US_DataIO::Scan* scn = &scanData[ scnx ];
158  bool use_stddev = ( scn->nz_stddev && scn->stddevs.size() > 0 );
159  return ( use_stddev ? scn->stddevs[ radx ] : 0.0 );
160 }
161 
162 // Calculate the average temperature value across scans
164 {
165  int scansize = scanData.size();
166  double sum = 0.0;
167 
168  for ( int ii = 0; ii < scansize; ii++ )
169  {
170  sum += scanData[ ii ].temperature;
171  }
172 
173  return ( sum / (double)scansize );
174 }
175 
176 // Calculate the spread of temperature values across scans
178 {
179  int scansize = scanData.size();
180  double smin = 99.0;
181  double smax = 0.0;
182 
183  for ( int ii = 0; ii < scansize; ii++ )
184  {
185  double temp = scanData[ ii ].temperature;
186  smin = qMin( smin, temp );
187  smax = qMax( smax, temp );
188  }
189 
190  return qAbs( smax - smin );
191 }
192 
193 bool US_DataIO::readLegacyFile( const QString& file,
194  BeckmanRawScan& data )
195 {
196  // Open the file for reading
197  QFile ff( file );
198  if ( ! ff.open( QIODevice::ReadOnly | QIODevice::Text ) ) return false;
199  QTextStream ts( &ff );
200 
201  // Read the description
202  data.description = ts.readLine();
203 
204  // Read scan parameters
205  QString sc = ts.readLine();
206  QStringList pp = sc.split( " ", QString::SkipEmptyParts );
207 
208  if ( pp.size() < 8 ) return false;
209 
210  data.type = pp[ 0 ].toAscii()[ 0 ]; // I P R W F
211  data.cell = pp[ 1 ].toInt();
212  data.temperature = pp[ 2 ].toDouble();
213  data.rpm = pp[ 3 ].toDouble();
214 //data.rpm = qRound( data.rpm / 50.0 ) * 50.0;
215  data.seconds = pp[ 4 ].toDouble();
216  data.omega2t = pp[ 5 ].toDouble();
217  data.rpoint = pp[ 6 ].toDouble();
218  data.count = pp[ 7 ].toInt();
219  data.nz_stddev = false;
220  // Round speed to nearest multiple of 100
221  data.rpm = qRound( data.rpm / 100.0 ) * 100.0;
222 
223 
224  // Read radius, data, and standard deviation
225  data.rvalues.clear();
226  data.stddevs.clear();
227  bool interference_data = ( data.type == 'P' );
228 
229  while ( ! ts.atEnd() )
230  {
231  sc = ts.readLine();
232 
233  pp = sc.split( " ", QString::SkipEmptyParts );
234  if ( pp.size() == 1 )
235  pp = sc.split( "\t", QString::SkipEmptyParts );
236 
237  double xval = pp[ 0 ].toDouble();
238  double rval = pp[ 1 ].toDouble();
239  double sval = 0.0;
240 
241  if ( ! interference_data && pp.size() > 2 )
242  {
243  sval = pp[ 2 ].toDouble();
244  if ( sval != 0.0 )
245  data.nz_stddev = true;
246  }
247 
248  data.rvalues << rval;
249  data.xvalues << xval;
250  data.stddevs << sval;
251  }
252 
253  if ( ! data.nz_stddev )
254  data.stddevs.clear();
255 
256  ff.close();
257  return true;
258 }
259 
260 int US_DataIO::writeRawData( const QString& file, RawData& data )
261 {
262  // Open the file for writing
263  QFile ff( file );
264  if ( ! ff.open( QIODevice::WriteOnly ) ) return CANTOPEN;
265  QDataStream ds( &ff );
266 
267  // Using quint32 to ensure same data size on all platforms
268  quint32 crc = 0xffffffffUL;
269 
270  // Write magic number
271  char magic[ 5 ] = "UCDA";
272  write( ds, magic, 4, crc );
273 
274  // Write format version
275  char fmt[ 3 ];
276  sprintf( fmt, "%2.2i", format_version );
277  write( ds, fmt, 2, crc );
278 
279  // Write data type
280  write( ds, data.type, 2, crc );
281 
282  // Write cell
283  write( ds, (const char*)&data.cell, 1, crc );
284 
285  // Write channel
286  write( ds, (const char*)&data.channel, 1, crc );
287 
288  // Create and write a guid
289  write( ds, data.rawGUID, 16, crc );
290 
291  // Write description
292  char desc[ 240 ];
293  memset( desc, '\0', sizeof desc ); // bzero is not defined in WIN32
294 
295  QByteArray dd = data.description.toLatin1();
296  strncpy( desc, dd.data(), sizeof desc );
297  write( ds, desc, sizeof desc, crc );
298 
299  // Find min and max radius, data, and std deviation
300  // First the radii
301  double min_radius = 1.0e99;
302  double max_radius = -1.0e99;
303 
304  for ( int ii = 0; ii < data.xvalues.size(); ii++ )
305  {
306  min_radius = qMin( min_radius, data.xvalues[ ii ] );
307  max_radius = qMax( max_radius, data.xvalues[ ii ] );
308  }
309 
310  // Now the data and SD
311  Parameters pp;
312 
313  pp.min_data1 = 1.0e99;
314  pp.max_data1 = -1.0e99;
315  pp.min_data2 = 1.0e99;
316  pp.max_data2 = -1.0e99;
317 
318  bool allz_stdd = true;
319  int scCount = data.scanCount();
320  int ptCount = data.pointCount();
321 
322  for ( int ii = 0; ii < scCount; ii++ )
323  {
324  Scan* sc = &data.scanData[ ii ];
325  bool nz_stddev = ( sc->nz_stddev && sc->stddevs.size() > 0 );
326 
327  for ( int jj = 0; jj < ptCount; jj++ )
328  {
329  pp.min_data1 = qMin( pp.min_data1, sc->rvalues[ jj ] );
330  pp.max_data1 = qMax( pp.max_data1, sc->rvalues[ jj ] );
331 
332  if ( nz_stddev )
333  {
334  pp.min_data2 = qMin( pp.min_data2, sc->stddevs[ jj ] );
335  pp.max_data2 = qMax( pp.max_data2, sc->stddevs[ jj ] );
336  allz_stdd = false;
337  }
338  }
339  }
340 
341  if ( allz_stdd )
342  {
343  pp.min_data2 = 0.0;
344  pp.max_data2 = 0.0;
345  }
346 
347  // Distance between radius entries
348  double r1 = data.xvalues[ 0 ];
349  double r2 = data.xvalues[ 1 ];
350 
351  // The unions below are a little tricky in order to get the
352  // c++ types and architecture endian-ness right. Floats
353  // are written to uf.f (union float - float ). Then the
354  // 4 bytes of the float are treated as a 32-bit unsigned int and
355  // converted to ui.u (union integer - unsigned ). Finally
356  // the ui.c type is to satify the c++ type for the write call.
357 
358  union
359  {
360  quint32 u;
361  float f;
362  } uf;
363 
364  union
365  {
366  char c[ 4 ];
367  uchar u[ 4 ];
368  } ui;
369 
370  uf.f = (float) min_radius; // min_radius
371  qToLittleEndian( uf.u, ui.u );
372  write( ds, ui.c, 4, crc );
373 
374  uf.f = (float) max_radius; // max_radius
375  qToLittleEndian( uf.u, ui.u );
376  write( ds, ui.c, 4, crc );
377 
378  uf.f = (float) ( r2 - r1 ); // delta r
379  qToLittleEndian( uf.u, ui.u );
380  write( ds, ui.c, 4, crc );
381 
382  uf.f = (float) pp.min_data1; // minimum data value
383  qToLittleEndian( uf.u, ui.u );
384  write( ds, ui.c, 4, crc );
385 
386  uf.f = (float) pp.max_data1; // maximum data value
387  qToLittleEndian( uf.u, ui.u );
388  write( ds, ui.c, 4, crc );
389 
390  uf.f = (float) pp.min_data2; // minimum std deviation value
391  qToLittleEndian( uf.u, ui.u );
392  write( ds, ui.c, 4, crc );
393 
394  uf.f = (float) pp.max_data2; // maximum std deviation value
395  qToLittleEndian( uf.u, ui.u );
396  write( ds, ui.c, 4, crc );
397 
398  // Write out scan count
399  qToLittleEndian( (quint16)data.scanData.size(), ui.u );
400  write( ds, ui.c, 2, crc );
401 
402  // Loop for each scan
403  for ( int ii = 0; ii < data.scanData.size(); ii++ )
404  writeScan( ds, data.scanData[ ii ], crc, pp );
405 
406  qToLittleEndian( crc, ui.u ); // crc
407  ds.writeRawData( ui.c, 4 );
408 
409  ff.close();
410 
411  return OK;
412 }
413 
414 void US_DataIO::writeScan( QDataStream& ds, const Scan& data,
415  quint32& crc, const Parameters& pp )
416 {
417  union
418  {
419  quint32 u;
420  float f;
421  } uf;
422 
423  union
424  {
425  char c[ 4 ];
426  uchar u[ 4 ];
427  } ui;
428 
429  quint16 si;
430 
431  char d[ 5 ] = "DATA";
432  write( ds, d, 4, crc );
433 
434  uf.f = (float) data.temperature; // scan temperature
435  qToLittleEndian( uf.u, ui.u );
436  write( ds, ui.c, 4, crc );
437 
438  uf.f = (float) data.rpm; // scan rpm
439  qToLittleEndian( uf.u, ui.u );
440  write( ds, ui.c, 4, crc );
441 
442  qToLittleEndian( (quint32)data.seconds, ui.u ); // scan time
443  write( ds, ui.c, 4, crc );
444 
445  uf.f = (float) data.omega2t; //scan omega^2 t
446  qToLittleEndian( uf.u, ui.u );
447  write( ds, ui.c, 4, crc );
448 
449  // Encoded wavelength
450  si = (quint16)qRound( data.wavelength * 10.0 );
451  qToLittleEndian<quint16>( si, ui.u );
452 //qDebug() << "DIO: wvln si" << data.wavelength << si
453 // << "ui.c" << (int)ui.c[0] << (int)ui.c[1];
454  write( ds, ui.c, 2, crc );
455 
456  uf.f = (float) data.delta_r; // delta r
457  qToLittleEndian( uf.u, ui.u );
458  write( ds, ui.c, 4, crc );
459 
460  quint32 valueCount = data.rvalues.size(); // number of values
461  qToLittleEndian( valueCount, ui.u );
462  write( ds, ui.c, 4, crc );
463 
464  // Write readings
465  double delta = ( pp.max_data1 - pp.min_data1 ) / 65535;
466  double delta2 = ( pp.max_data2 - pp.min_data2 ) / 65535;
467 
468  bool stdDev = ( ( pp.min_data2 != 0.0 || pp.max_data2 != 0.0 )
469  && ( data.stddevs.size() == data.rvalues.size() ) );
470 
471  for ( int ii = 0; ii < data.rvalues.size(); ii++ )
472  {
473  double rval = data.rvalues[ ii ];
474  // reading
475  si = (quint16) qRound( ( rval - pp.min_data1 ) / delta );
476  qToLittleEndian( si, ui.u );
477  write( ds, ui.c, 2, crc );
478 
479  // If applicable, write std deviation
480  if ( stdDev )
481  {
482  double sval = data.stddevs[ ii ];
483  si = (quint16) qRound( ( sval - pp.min_data2 ) / delta2 );
484  qToLittleEndian( si, ui.u );
485  write( ds, ui.c, 2, crc );
486  }
487  }
488 
489  // Write interpolated flags
490  int flagSize = ( valueCount + 7 ) / 8;
491  write( ds, data.interpolated.data(), flagSize, crc );
492 }
493 
494 void US_DataIO::write( QDataStream& ds, const char* c, int len, quint32& crc )
495 {
496  ds.writeRawData( c, len );
497  crc = US_Crc::crc32( crc, (unsigned char*) c, len );
498 }
499 
500 int US_DataIO::readRawData( const QString& file, RawData& data )
501 {
502  QFile ff( file );
503  if ( ! ff.open( QIODevice::ReadOnly ) ) return CANTOPEN;
504  QDataStream ds( &ff );
505 
506  int err = OK;
507  quint32 crc = 0xffffffffUL;
508 
509  try
510  {
511  // Read magic number
512  char magic[ 4 ];
513  read( ds, magic, 4, crc );
514  if ( strncmp( magic, "UCDA", 4 ) != 0 ) throw NOT_USDATA;
515 
516  // Check the version number
517  unsigned char ver[ 2 ];
518  read( ds, (char*) ver, 2, crc );
519  quint32 version = ( ( ver[ 0 ] & 0x0f ) << 8 ) | ( ver[ 1 ] & 0x0f );
520 // if ( version != format_version ) throw BAD_VERSION;
521  if ( version > format_version ) throw BAD_VERSION;
522 
523  bool wvlf_new = ( version > (quint32)4 );
524 
525  // Read and get the file type
526  char type[ 3 ];
527  read( ds, type, 2, crc );
528  type[ 2 ] = '\0';
529 
530  QStringList types = QStringList() << "RA" << "IP" << "RI" << "FI"
531  << "WA" << "WI";
532 
533  if ( ! types.contains( QString( type ) ) ) throw BADTYPE;
534  strncpy( data.type, type, 2 );
535 
536  // Get the cell
537  union
538  { char c[ 4 ];
539  int i;
540  } cell;
541 
542  cell.i = 0;
543 
544  read( ds, cell.c, 1, crc );
545  data.cell = qFromLittleEndian( cell.i );
546 
547  // Get the channel
548  read( ds, &data.channel, 1, crc );
549 
550  // Get the guid
551  read( ds, data.rawGUID, 16, crc );
552 
553  // Get the description
554  char desc[ 240 ];
555  read( ds, desc, 240, crc );
556  data.description = QString( desc );
557 
558  // Get the parameters to expand the values
559  union
560  {
561  char c[ 2 ];
562  quint16 I;
563  } si;
564 
565  union
566  {
567  char c[ 4 ];
568  qint32 I;
569  float f;
570  } u1;
571 
572  union
573  {
574  char c[ 4 ];
575  qint32 I;
576  float f;
577  } u2;
578 
579  double min_radius;
580 
581  read( ds, u1.c, 4, crc );
582  u2.I = qFromLittleEndian( u1.I );
583  min_radius = u2.f;
584 
585  read( ds, u1.c, 4, crc );
586  // Unused
587  //u2.I = qFromLittleEndian( u1.I );
588  //min_radius = u2.f;
589 
590  read( ds, u1.c, 4, crc );
591  u2.I = qFromLittleEndian( u1.I );
592  double delta_radius = u2.f;
593 
594  read( ds, u1.c, 4, crc );
595  u2.I = qFromLittleEndian( u1.I );
596  double min_data1 = u2.f;
597 
598  read( ds, u1.c, 4, crc );
599  u2.I = qFromLittleEndian( u1.I );
600  double max_data1 = u2.f;
601 
602  read( ds, u1.c, 4, crc );
603  u2.I = qFromLittleEndian( u1.I );
604  double min_data2 = u2.f;
605 
606  read( ds, u1.c, 4, crc );
607  u2.I = qFromLittleEndian( u1.I );
608  double max_data2 = u2.f;
609 
610  read( ds, si.c, 2, crc );
611  qint16 scan_count = qFromLittleEndian( si.I );
612 
613  // Read each scan
614  int valueCount = 0;
615  for ( int ii = 0 ; ii < scan_count; ii ++ )
616  {
617  read( ds, u1.c, 4, crc );
618  if ( strncmp( u1.c, "DATA", 4 ) != 0 ) throw NOT_USDATA;
619 
620  Scan sc;
621 
622  // Temperature
623  read( ds, u1.c, 4, crc );
624  u2.I = qFromLittleEndian( u1.I );
625  sc.temperature = u2.f;
626 
627  // RPM
628  read( ds, u1.c, 4, crc );
629  u2.I = qFromLittleEndian( u1.I );
630  sc.rpm = u2.f;
631 //sc.rpm = qRound( sc.rpm / 50.0 ) * 50.0;
632  // Round speed to nearest multiple of 100
633  sc.rpm = qRound( sc.rpm / 100.0 ) * 100.0;
634 
635  // Seconds
636  read( ds, u1.c, 4, crc );
637  sc.seconds = qFromLittleEndian( u1.I );
638 
639  // Omega2t
640  read( ds, u1.c, 4, crc );
641  u2.I = qFromLittleEndian( u1.I );
642  sc.omega2t = u2.f;
643 
644  // Wavelength
645  read( ds, si.c, 2, crc );
646  if ( wvlf_new )
647  sc.wavelength = qFromLittleEndian( si.I ) / 10.0;
648  else
649  sc.wavelength = qFromLittleEndian( si.I ) / 100.0 + 180.0;
650 
651  // Delta_r
652  read( ds, u1.c, 4, crc );
653  u2.I = qFromLittleEndian( u1.I );
654  sc.delta_r = u2.f;
655 
656  // Reading count
657  read( ds, u1.c, 4, crc );
658  valueCount = qFromLittleEndian( u1.I );
659 
660  // Get the readings
661  double factor1 = ( max_data1 - min_data1 ) / 65535.0;
662  double factor2 = ( max_data2 - min_data2 ) / 65535.0;
663  bool stdDev = ( min_data2 != 0.0 || max_data2 != 0.0 );
664 
665  for ( int jj = 0; jj < valueCount; jj++ )
666  {
667  double rval;
668  double sval;
669 
670  read( ds, si.c, 2, crc );
671  rval = qFromLittleEndian( si.I ) * factor1 + min_data1;
672 
673  if ( stdDev )
674  {
675  read( ds, si.c, 2, crc );
676  sval = qFromLittleEndian( si.I ) * factor2 + min_data2;
677  }
678  else
679  sval = 0.0;
680 
681  // Add the reading to the scan
682  sc.rvalues << rval;
683  sc.stddevs << sval;
684  }
685 
686  if ( !stdDev )
687  {
688  sc.stddevs.clear();
689  sc.nz_stddev = false;
690  }
691  else
692  sc.nz_stddev = true;
693 
694  // Get the interpolated bitmap;
695  int bytes = ( valueCount + 7 ) / 8;
696  char* interpolated = new char[ bytes ];
697 
698  read( ds, interpolated, bytes, crc );
699 
700  sc.interpolated = QByteArray( interpolated, bytes );
701 
702  delete [] interpolated;
703 
704  // Add the scan to the data
705  data.scanData << sc;
706  }
707 
708  // Calculate the radius vector
709  data.xvalues.clear();
710  double radius = min_radius;
711 
712  for ( int jj = 0; jj < valueCount; jj++ )
713  {
714  data.xvalues << radius;
715  radius += delta_radius;
716  }
717 
718  // Read the crc
719  quint32 read_crc;
720  ds.readRawData( (char*) &read_crc , 4 );
721  if ( crc != qFromLittleEndian( read_crc ) ) throw BADCRC;
722 
723  } catch( ioError error )
724  {
725  err = error;
726  }
727 
728  ff.close();
729  return err;
730 }
731 
732 void US_DataIO::read( QDataStream& ds, char* cc, int len, quint32& crc )
733 {
734  ds.readRawData( cc, len );
735  crc = US_Crc::crc32( crc, (uchar*) cc, len );
736 }
737 
738 int US_DataIO::readEdits( const QString& filename, EditValues& parameters )
739 {
740  QFile ff( filename );
741  if ( ! ff.open( QIODevice::ReadOnly ) ) return CANTOPEN;
742 
743  QXmlStreamReader xml( &ff );
744 
745  while ( ! xml.atEnd() )
746  {
747  xml.readNext();
748 
749  if ( xml.isStartElement() )
750  {
751  if ( xml.name() == "identification" )
752  ident ( xml, parameters );
753 
754  else if ( xml.name() == "run" )
755  run( xml, parameters );
756 
757  else if ( xml.name() == "experiment" )
758  {
759  QXmlStreamAttributes a = xml.attributes();
760  parameters.expType = a.value( "type" ).toString();
761  }
762  }
763  }
764 
765  bool error = xml.hasError();
766  ff.close();
767 
768  if ( error ) return BADXML;
769 
770  return OK;
771 }
772 
773 void US_DataIO::ident( QXmlStreamReader& xml, EditValues& parameters )
774 {
775  while ( ! xml.atEnd() )
776  {
777  if ( xml.isEndElement() && xml.name() == "identification" ) return;
778 
779  if ( xml.isStartElement() && xml.name() == "runid" )
780  {
781  QXmlStreamAttributes a = xml.attributes();
782  parameters.runID = a.value( "value" ).toString();
783  }
784 
785  if ( xml.isStartElement() && xml.name() == "editGUID" )
786  {
787  QXmlStreamAttributes a = xml.attributes();
788  parameters.editGUID = a.value( "value" ).toString();
789  }
790 
791  if ( xml.isStartElement() && xml.name() == "rawDataGUID" )
792  {
793  QXmlStreamAttributes a = xml.attributes();
794  parameters.dataGUID = a.value( "value" ).toString();
795  }
796 
797  xml.readNext();
798  }
799 }
800 
801 void US_DataIO::run( QXmlStreamReader& xml, EditValues& parameters )
802 {
803  QXmlStreamAttributes a = xml.attributes();
804  parameters.cell = a.value( "cell" ).toString();
805  parameters.channel = a.value( "channel" ).toString();
806  parameters.wavelength = a.value( "wavelength" ).toString();
807 
808  if ( parameters.wavelength.contains( "-" ) )
809  all_lambdas( xml, parameters );
810 
811  while ( ! xml.atEnd() )
812  {
813  if ( xml.isEndElement() && xml.name() == "run" ) return;
814 
815  if ( xml.isStartElement() && xml.name() == "parameters" )
816  params( xml, parameters );
817 
818  if ( xml.isStartElement() && xml.name() == "operations" )
819  operations( xml, parameters );
820 
821  if ( xml.isStartElement() && xml.name() == "edited" )
822  do_edits( xml, parameters );
823 
824  if ( xml.isStartElement() && xml.name() == "excludes" )
825  excludes( xml, parameters );
826 
827  xml.readNext();
828  }
829 }
830 
831 void US_DataIO::all_lambdas( QXmlStreamReader& xml, EditValues& parameters )
832 {
833  while ( ! xml.atEnd() )
834  {
835  if ( xml.isEndElement() && xml.name() == "lambdas" ) return;
836 
837  if ( xml.isStartElement() && xml.name() == "lambda" )
838  {
839  QXmlStreamAttributes a = xml.attributes();
840  parameters.lambdas << a.value( "value" ).toString().toInt();
841  }
842 
843  xml.readNext();
844  }
845 }
846 
847 void US_DataIO::params( QXmlStreamReader& xml, EditValues& parameters )
848 {
849  parameters.speedData.clear();
850  bool isEquil = ( parameters.expType == "Equilibrium" );
851  int spx = 0;
852 
853  while ( ! xml.atEnd() )
854  {
855  if ( xml.isEndElement() && xml.name() == "parameters" ) return;
856 
857  else if ( !xml.isStartElement() )
858  {
859  if ( xml.isEndElement() && xml.name() == "speed" )
860  spx++;
861 
862  xml.readNext();
863  continue;
864  }
865 
866  else if ( xml.name() == "speed" )
867  {
868  QXmlStreamAttributes a = xml.attributes();
869  parameters.speedData << SpeedData();
870  parameters.speedData[ spx ].speed
871  = a.value( "value" ).toString().toDouble();
872  parameters.speedData[ spx ].first_scan
873  = a.value( "scanStart" ).toString().toInt();
874  parameters.speedData[ spx ].scan_count
875  = a.value( "scanCount" ).toString().toInt();
876  }
877 
878  else if ( xml.name() == "meniscus" )
879  {
880  QXmlStreamAttributes a = xml.attributes();
881  parameters.meniscus = a.value( "radius" ).toString().toDouble();
882 
883  if ( isEquil )
884  parameters.speedData[ spx ].meniscus = parameters.meniscus;
885  }
886 
887  else if ( xml.name() == "plateau" )
888  {
889  QXmlStreamAttributes a = xml.attributes();
890  parameters.plateau = a.value( "radius" ).toString().toDouble();
891  }
892 
893  else if ( xml.name() == "air_gap" )
894  {
895  QXmlStreamAttributes a = xml.attributes();
896  parameters.airGapLeft = a.value( "left" ).toString().toDouble();
897  parameters.airGapRight = a.value( "right" ).toString().toDouble();
898  parameters.gapTolerance = a.value( "tolerance" ).toString().toDouble();
899  }
900 
901  else if ( xml.name() == "baseline" )
902  {
903  QXmlStreamAttributes a = xml.attributes();
904  parameters.baseline = a.value( "radius" ).toString().toDouble();
905  }
906 
907  else if ( xml.name() == "data_range" )
908  {
909  QXmlStreamAttributes a = xml.attributes();
910  parameters.rangeLeft = a.value( "left" ).toString().toDouble();
911  parameters.rangeRight = a.value( "right" ).toString().toDouble();
912 
913  if ( isEquil )
914  {
915  parameters.speedData[ spx ].dataLeft = parameters.rangeLeft;
916  parameters.speedData[ spx ].dataRight = parameters.rangeRight;
917  }
918  }
919 
920  else if ( xml.name() == "od_limit" )
921  {
922  QXmlStreamAttributes a = xml.attributes();
923  parameters.ODlimit = a.value( "value" ).toString().toDouble();
924  }
925 
926  xml.readNext();
927  }
928 }
929 
930 void US_DataIO::operations( QXmlStreamReader& xml, EditValues& parameters )
931 {
932  while ( ! xml.atEnd() )
933  {
934  if ( xml.isEndElement() && xml.name() == "operations" ) return;
935 
936  if ( xml.isStartElement() && xml.name() == "invert" )
937  parameters.invert = -1.0;
938 
939  if ( xml.isStartElement() && xml.name() == "remove_spikes" )
940  parameters.removeSpikes = true;
941 
942  if ( xml.isStartElement() && xml.name() == "floating_data" )
943  parameters.floatingData = true;
944 
945  if ( xml.isStartElement() && xml.name() == "subtract_ri_noise" )
946  {
947  QXmlStreamAttributes a = xml.attributes();
948  parameters.noiseOrder = a.value( "order" ).toString().toInt();
949  }
950 
951  xml.readNext();
952  }
953 }
954 
955 void US_DataIO::excludes( QXmlStreamReader& xml, EditValues& parameters )
956 {
957  while ( ! xml.atEnd() )
958  {
959  if ( xml.isEndElement() && xml.name() == "excludes" ) return;
960 
961  if ( xml.isStartElement() && xml.name() == "exclude" )
962  {
963  QXmlStreamAttributes a = xml.attributes();
964  parameters.excludes << a.value( "scan" ).toString().toInt();
965  }
966 
967  xml.readNext();
968  }
969 }
970 
971 void US_DataIO::do_edits( QXmlStreamReader& xml, EditValues& parameters )
972 {
973  while ( ! xml.atEnd() )
974  {
975  if ( xml.isEndElement() && xml.name() == "edited" ) return;
976 
977  if ( xml.isStartElement() && xml.name() == "edit" )
978  {
979  EditedPoint e;
980  QXmlStreamAttributes a = xml.attributes();
981  e.scan = a.value( "scan" ).toString().toInt();
982  e.radius = a.value( "radius" ).toString().toInt();
983  e.value = a.value( "value" ).toString().toDouble();
984 
985  parameters.editedPoints << e;
986  }
987 
988  xml.readNext();
989  }
990 }
991 
992 int US_DataIO::loadData( const QString& directory,
993  const QString& editFilename,
994  QVector< EditedData >& data )
995 {
996  QVector< RawData > raw;
997 
998  return loadData( directory, editFilename, data, raw );
999 }
1000 
1001 int US_DataIO::loadData( const QString& directory,
1002  const QString& editFilename,
1003  EditedData& data )
1004 {
1005  QVector< RawData > raw;
1006  QVector< EditedData > editedData;
1007 
1008  int result = loadData( directory, editFilename, editedData, raw );
1009 
1010  if ( result == OK )
1011  data = editedData[ 0 ];
1012 
1013  return result;
1014 }
1015 
1016 // Load edited and raw data
1017 int US_DataIO::loadData( const QString& directory,
1018  const QString& editFilename,
1019  QVector< EditedData >& data,
1020  QVector< RawData >& raw )
1021 {
1022  QString ftriple = editFilename.section( ".", -2, -2 );
1023  // Determine raw file name by removing editID
1024  QString rawDataFile = editFilename;
1025  QString edtFileRead = editFilename;
1026  QString filepart1 = editFilename.section( ".", 0, -7 );
1027  QString filepart2 = editFilename.section( ".", -5, -3 );
1028  rawDataFile = filepart1 + "." + filepart2 + ".";
1029  QString clambda = ftriple .section( ".", -1, -1 );
1030  bool isMwl = clambda.contains( "-" );
1031 
1032  if ( isMwl )
1033  {
1034  QString elambda = clambda .section( "@", -2, -2 );
1035  clambda = clambda .section( "@", -1, -1 );
1036  edtFileRead = editFilename.section( ".", 0, -3 )
1037  + "." + elambda + ".xml";
1038  }
1039 
1040  rawDataFile = rawDataFile + clambda + ".auc";
1041 
1042 //qDebug() << "dIO:ldEd: editFilename" << editFilename;
1043 //qDebug() << "dIO:ldEd: rawDataFile" << rawDataFile;
1044 //qDebug() << "dIO:ldEd: edtFileRead" << edtFileRead;
1045 
1046  // Get the raw data
1047  RawData dd;
1048  ioError result = (ioError)readRawData( directory + "/" + rawDataFile, dd );
1049  if ( result != OK ) throw result;
1050  raw << dd;
1051  qApp->processEvents();
1052 
1053  // Get the edit data
1054  EditValues ev;
1055  result = (ioError)readEdits( directory + "/" + edtFileRead, ev );
1056  if ( result != OK ) throw result;
1057 
1058  // Check for uuid match
1059  QString rawGuid = US_Util::uuid_unparse( (uchar*)dd.rawGUID );
1060 
1061  if ( rawGuid != ev.dataGUID )
1062  {
1063  QString clambda = edtFileRead.section( ".", -2, -2 );
1064  if ( clambda.contains( "-" ) )
1065  ev.dataGUID = rawGuid;
1066  else
1067  throw NO_GUID_MATCH;
1068  }
1069 
1070  // Apply the edits
1071  EditedData ed;
1072 
1073  QStringList sl = editFilename.split( "." );
1074 
1075  ed.runID = sl[ 0 ];
1076  ed.editID = sl[ 1 ];
1077  ed.dataType = sl[ 2 ];
1078  ed.cell = sl[ 3 ];
1079  ed.channel = sl[ 4 ];
1080  ed.wavelength = clambda;
1081  ed.description = dd.description;
1082  ed.expType = ev.expType;
1083  ed.dataGUID = ev.dataGUID;
1084  ed.editGUID = ev.editGUID;
1085  ed.meniscus = ev.meniscus;
1086  ed.plateau = ev.plateau;
1087  ed.baseline = ev.baseline;
1088  ed.ODlimit = ( ed.dataType == "RA" || ed.dataType == "RI" ) ?
1089  ev.ODlimit : 1e+99;
1090  ed.floatingData = ev.floatingData;
1091 //qDebug() << "dIO:ldEd: ed.descr" << ed.description
1092 // << "ed.wavelength" << ed.wavelength << "ed.dataType ed.ODlimit"
1093 // << ed.dataType << ed.ODlimit;
1094 
1095  if ( ed.expType == "Equilibrium" )
1096  ed.speedData << ev.speedData;
1097 
1098  // Invert values before updating edited points
1099  if ( ev.invert < 0 )
1100  {
1101  for ( int ii = 0; ii < dd.scanData.size(); ii++ )
1102  {
1103  Scan* sc = &dd.scanData[ ii ];
1104  for ( int jj = 0; jj < sc->rvalues.size(); jj++ )
1105  {
1106  sc->rvalues[ jj ] *= ev.invert;
1107  }
1108  }
1109  }
1110 
1111  // Update any edited points
1112  for ( int ii = 0; ii < ev.editedPoints.length(); ii++ )
1113  {
1114  int scnx = ev.editedPoints[ ii ].scan;
1115  int radx = ev.editedPoints[ ii ].radius;
1116  double value = ev.editedPoints[ ii ].value;
1117  dd.setValue( scnx, radx, value );
1118  }
1119 
1120  // Update for interference data
1121  if ( ed.dataType == "IP" )
1122  {
1123  adjust_interference( dd, ev ); // rawData, editValues
1124  calc_integral ( dd, ev );
1125  }
1126 
1127  // Do not copy excluded data or data outside the edit range
1128  for ( int ii = 0; ii < dd.scanData.size(); ii++ )
1129  {
1130  if ( ev.excludes.contains( ii ) ) continue;
1131 
1132  Scan sc;
1133  copyRange( ev.rangeLeft, ev.rangeRight, dd.scanData[ ii ],
1134  sc, dd.xvalues );
1135 
1136  ed.scanData << sc;
1137  }
1138 
1139  // Only need to copy radius vector for given range once
1140  copyxRange( ev.rangeLeft, ev.rangeRight, dd.xvalues, ed.xvalues );
1141 
1142  // Determine plateau values for each scan
1143  for ( int ii = 0; ii < ed.scanData.size(); ii++ )
1144  {
1145  int point = ed.xindex( ed.plateau );
1146  ed.scanData[ ii ].plateau = ed.scanData[ ii ].rvalues[ point ];
1147  }
1148 
1149  if ( ev.removeSpikes )
1150  {
1151  double smoothed_value;
1152 
1153  // For each scan
1154  for ( int ii = 0; ii < ed.scanData.size(); ii++ )
1155  {
1156  Scan* sc = &ed.scanData [ ii ];
1157  int start = 0;
1158  int end = sc->rvalues.size() - 1; // index of the last one
1159 
1160  for ( int jj = start; jj < end; jj++ )
1161  {
1162  if ( spike_check( *sc, ed.xvalues, jj, start, end,
1163  &smoothed_value ) )
1164  {
1165  sc->rvalues[ jj ] = smoothed_value;
1166 
1167  // If previous consecutive points are interpolated, then
1168  // redo them
1169  int index = jj - 1;
1170  unsigned char cc = sc->interpolated[ index / 8 ];
1171 
1172  while ( cc & ( 1 << ( 7 - index % 8 ) ) )
1173  {
1174  if ( spike_check( *sc, ed.xvalues, index, start, end,
1175  &smoothed_value ) )
1176  sc->rvalues[ index ] = smoothed_value;
1177 
1178  index--;
1179  cc = sc->interpolated[ index / 8 ];
1180  }
1181  }
1182  }
1183  }
1184  }
1185 
1186  if ( ev.noiseOrder > 0 )
1187  {
1188  QList< double > residuals = calc_residuals( ev.noiseOrder, ed.scanData );
1189 
1190  for ( int ii = 0; ii < ed.scanData.size(); ii++ )
1191  {
1192  for ( int jj = 0; jj < ed.scanData[ ii ].rvalues.size(); jj++ )
1193  ed.scanData[ ii ].rvalues[ jj ] -= residuals[ ii ];
1194  }
1195  }
1196 
1197  data << ed;
1198  return OK;
1199 }
1200 
1201 // Adjust interference data
1203 {
1204  // Find first scan
1205  for ( int ii = 0; ii < data.scanData.size(); ii++ )
1206  {
1207  if ( ev.excludes.contains( ii ) ) continue;
1208 
1209  US_DataIO::Scan* sc;
1210 
1211  int r_left = index( data.xvalues, ev.airGapLeft );
1212  int r_right = index( data.xvalues, ev.airGapRight );
1213  double sum = 0.0;
1214 
1215  for ( int kk = r_left; kk <= r_right; kk++ )
1216  sum += data.reading( ii, kk );
1217 
1218  double average = sum / ( r_right - r_left + 1 );
1219 
1220  for ( int jj = ii + 1; jj < data.scanData.size(); jj++ )
1221  {
1222  // Get average difference between gap in first included
1223  // scan and each subsequent scan
1224 
1225  sc = &data.scanData[ jj ];
1226  sum = 0.0;
1227 
1228  for ( int kk = r_left; kk <= r_right; kk++ )
1229  sum += data.reading( jj, kk ) - data.reading( ii, kk );
1230 
1231  double delta = sum / ( r_right - r_left + 1 );
1232 
1233  // Subtract average difference from all points
1234  for ( int kk = 0; kk < sc->rvalues.size(); kk++ )
1235  sc->rvalues[ kk ] -= delta;
1236  }
1237 
1238  for ( int jj = ii; jj < data.scanData.size(); jj++ )
1239  {
1240  sc = &data.scanData[ jj ];
1241 
1242  for ( int kk = 0; kk < sc->rvalues.size(); kk++ )
1243  sc->rvalues[ kk ] -= average;
1244  }
1245 
1246  return; // After first included scan
1247  }
1248 }
1249 
1250 // Calculate and apply an integration to readings data
1252 {
1253  // This function gets a little tricky because we have raw data, but
1254  // want to adjust in the range identified by the user *and*
1255  // not use excluded scans.
1256 
1257  int kk = 0;
1258  QVector< int > included;
1259  QVector< double > integral;
1260  double val_min = 1.e+99;
1261 
1262  for ( int scx = 0; scx < data.scanData.size(); scx++ )
1263  {
1264  if ( ee.excludes.contains( scx ) ) continue;
1265 
1266  included << scx;
1267 
1268  integral << 0.0;
1269 
1270  // Arbitrarily add 1000 fringes to each reading value
1271  // to make sure we don't sum negatives. This is not needed from a
1272  // mathematical point of view, but physically, negative fringes
1273  // do not make sense.
1274  US_DataIO::Scan* sc = &data.scanData[ scx ];
1275 
1276  int r_left = index( data.xvalues, ee.rangeLeft );
1277  int r_right = index( data.xvalues, ee.rangeRight );
1278 
1279  for ( int rr = r_left; rr <= r_right; rr++ )
1280  integral[ kk ] += data.reading( scx, rr ) + 1000.0;
1281 
1282  kk++;
1283 
1284  if ( scx == 0 )
1285  {
1286  for ( int rr = r_left; rr <= r_right; rr++ )
1287  val_min = qMin( val_min, sc->rvalues[ rr ] );
1288  }
1289  }
1290 
1291  // Integral fringe shifts contribute exactly ( r_right - r_left + 1 )
1292  // to the integral, since we use unity stepsize in integral calculation.
1293 
1294  for ( int scx = 1; scx < included.size(); scx++ )
1295  {
1296  US_DataIO::Scan* sc = &data.scanData[ included[ scx ] ];
1297 
1298  int r_left = index( data.xvalues, ee.rangeLeft );
1299  int r_right = index( data.xvalues, ee.rangeRight );
1300  int current = scx;
1301  int previous = scx - 1;
1302  int position = 0;
1303  double points = (double) ( r_right - r_left + 1 );
1304 
1305  while ( integral[ current ] <= integral[ previous ] )
1306  {
1307  integral[ current ] += points;
1308  position++;
1309  }
1310 
1311  while ( integral[ current ] > integral[ previous ] )
1312  {
1313  integral[ current ] -= points;
1314  position--;
1315  }
1316 
1317  // Add the integral steps (which may be negative!) to each datapoint
1318  for ( int rr = r_left; rr <= r_right; rr++ )
1319  sc->rvalues[ rr ] += (double) position;
1320 
1321  double diff1 = integral[ previous ] - integral[ current ];
1322 
1323  double diff2 = integral[ previous ] - ( integral[ current ] + points );
1324 
1325  // The scan is one fringe too low
1326  if ( fabs( diff2 / diff1 ) < ee.gapTolerance )
1327  {
1328  // add one fringe to all readings
1329  for ( int rr = r_left; rr <= r_right; rr++ )
1330  sc->rvalues[ rr ] += 1.0;
1331 
1332  // Update integral for this scan
1333  integral[ current ] += points;
1334  }
1335 
1336  // Accumulate minimum reading after integral adjustment
1337  for ( int rr = r_left; rr <= r_right; rr++ )
1338  val_min = qMin( val_min, sc->rvalues[ rr ] );
1339  }
1340 
1341  // Apply a bias to the data so that its minimum is above zero
1342  if ( val_min < 0.0 )
1343  {
1344  double val_bias = (double)qCeil( -val_min );
1345 
1346  for ( int scx = 0; scx < included.size(); scx++ )
1347  {
1348  US_DataIO::Scan* sc = &data.scanData[ included[ scx ] ];
1349 
1350  int r_left = index( data.xvalues, ee.rangeLeft );
1351  int r_right = index( data.xvalues, ee.rangeRight );
1352 
1353  for ( int rad = r_left; rad <= r_right; rad++ )
1354  {
1355  sc->rvalues[ rad ] += val_bias;
1356  }
1357  }
1358  }
1359 }
1360 
1361 // Copy a range of readings values from one data set to another
1362 void US_DataIO::copyRange ( double left, double right, const Scan& orig,
1363  Scan& dest, const QVector< double >& origx )
1364 {
1365  dest.temperature = orig.temperature;
1366  dest.rpm = orig.rpm;
1367  dest.seconds = orig.seconds;
1368  dest.omega2t = orig.omega2t;
1369  dest.wavelength = orig.wavelength;
1370  dest.delta_r = orig.delta_r;
1371  dest.nz_stddev = orig.nz_stddev;
1372  dest.stddevs.clear();
1373 
1374  int index_L = index( origx, left );
1375  int index_R = index( origx, right );
1376 
1377  dest.interpolated.fill( (char)0, ( index_R - index_L ) / 8 + 1 );
1378 
1379  int current_bit = 0;
1380 
1381  for ( int ii = index_L; ii <= index_R; ii++ )
1382  {
1383  // Copy the concentration readings
1384  dest.rvalues << orig.rvalues[ ii ];
1385 
1386  // Copy the standard deviations if any are non-zero
1387  if ( orig.nz_stddev )
1388  dest.stddevs << orig.stddevs[ ii ];
1389 
1390  // Set the interpolated bits as needed
1391  unsigned char old_bit = (unsigned char)( 1 << ( 7 - ii % 8 ) );
1392 
1393  if ( ( orig.interpolated[ ii / 8 ] & old_bit ) != 0 )
1394  {
1395  unsigned char new_bit = (unsigned char)( 1 << ( 7 - current_bit % 8 ) );
1396  unsigned char byte = dest.interpolated[ current_bit / 8 ];
1397  dest.interpolated[ current_bit / 8 ] = byte | new_bit;
1398  }
1399 
1400  current_bit++;
1401  }
1402 
1403  if ( orig.nz_stddev )
1404  { // Likely some stddevs non-zero, but double-check since range changed
1405  int nnz = 0;
1406 
1407  for ( int ii = 0; ii < dest.stddevs.count(); ii++ )
1408  if ( dest.stddevs[ ii ] != 0.0 )
1409  nnz++;
1410 
1411  dest.nz_stddev = ( nnz > 0 );
1412  }
1413 }
1414 
1415 // Copy a range of X (radius) values from one data set to another
1416 void US_DataIO::copyxRange( double left, double right,
1417  const QVector< double >& origx,
1418  QVector< double >& destx )
1419 {
1420  for ( int ii = index( origx, left ); ii <= index( origx, right ); ii++ )
1421  { // Copy all X values within the specified range
1422  destx << origx[ ii ];
1423  }
1424 }
1425 
1426 // Determine if a readings value is a spike
1428  const QVector< double >& xvalues,
1429  int point, int start, int end, double* value )
1430 {
1431  static double rr[ 20 ]; // Spare room -- normally 10
1432  static double vv[ 20 ];
1433 
1434  double* xx = &rr[ 0 ];
1435  double* yy = &vv[ 0 ];
1436 
1437  double slope;
1438  double intercept;
1439  double correlation;
1440  double sigma = 0.0;
1441  int count = 0;
1442  int index;
1443 
1444  const double threshold = 10.0;
1445  unsigned char cc;
1446  unsigned char interpolated;
1447 
1448  // Only use non-interpolated points for the line fit
1449  // Decide to search left or right first.
1450  if ( point < ( start + end ) / 2 )
1451  {
1452  index = point - 1;
1453 
1454  // Search left for five points if we can
1455  while ( count < 5 )
1456  {
1457  if ( index < start ) break;
1458 
1459  cc = sc.interpolated[ index / 8 ];
1460  interpolated = (unsigned char)( cc & ( 1 << ( 7 - index % 8 ) ) );
1461 
1462  if ( ! interpolated )
1463  {
1464  rr[ count ] = xvalues [ index ];
1465  vv[ count ] = sc.rvalues[ index ];
1466  count++;
1467  }
1468 
1469  index--;
1470  }
1471 
1472  // Now search right
1473  index = point + 1;
1474 
1475  while ( count < 10 )
1476  {
1477  if ( index >= end ) break;
1478 
1479  cc = sc.interpolated[ index / 8 ];
1480  interpolated = (unsigned char)( cc & ( 1 << ( 7 - index % 8 ) ) );
1481 
1482  if ( ! interpolated )
1483  {
1484  rr[ count ] = xvalues [ index ];
1485  vv[ count ] = sc.rvalues[ index ];
1486  count++;
1487  }
1488 
1489  index++;
1490  }
1491  }
1492  else
1493  {
1494  index = point + 1;
1495 
1496  // Search right first
1497  while ( count < 5 )
1498  {
1499  if ( index >= end ) break;
1500 
1501  cc = sc.interpolated[ index / 8 ];
1502  interpolated = (unsigned char)( cc & ( 1 << ( 7 - index % 8 ) ) );
1503 
1504  if ( ! interpolated )
1505  {
1506  rr[ count ] = xvalues [ index ];
1507  vv[ count ] = sc.rvalues[ index ];
1508  count++;
1509  }
1510 
1511  index++;
1512  }
1513 
1514  // Now search left
1515  index = point - 1;
1516 
1517  while ( count < 10 )
1518  {
1519  if ( index < start ) break;
1520 
1521  cc = sc.interpolated[ index / 8 ];
1522  interpolated = (unsigned char)( cc & ( 1 << ( 7 - index % 8 ) ) );
1523 
1524  if ( ! interpolated )
1525  {
1526  rr[ count ] = xvalues [ index ];
1527  vv[ count ] = sc.rvalues[ index ];
1528  count++;
1529  }
1530 
1531  index--;
1532  }
1533  }
1534 
1535  US_Math2::linefit( &xx, &yy, &slope, &intercept, &sigma,
1536  &correlation, count );
1537 
1538  // If there is too much difference, it is a spike
1539  double current = sc.rvalues[ point ];
1540  double radius = xvalues [ point ];
1541  double projected = slope * radius + intercept;
1542 
1543  if ( fabs( projected - current ) > threshold * sigma )
1544  {
1545  // Interpolate
1546  *value = projected;
1547  return true;
1548  }
1549 
1550  return false; // Not a spike
1551 }
1552 
1553 QList< double > US_DataIO::calc_residuals( int order,
1554  const QVector< Scan >& sl )
1555 {
1556  int scan_count = sl.size();
1557 
1558  QVector< double > cov( order );
1559  QVector< double > aiv( scan_count );
1560  QVector< double > fiv( scan_count );
1561  QVector< double > tmv( scan_count );
1562  double* coeffs = cov.data();
1563  double* absorbance_integral = aiv.data();
1564  double* scan_time = tmv.data();
1565 
1566  // Calculate the integral of each scan which is needed for the least-squares
1567  // polynomial fit to correct for radially invariant baseline noise. We also
1568  // keep track of the total integral at each point.
1569 
1570  for ( int ii = 0; ii < scan_count; ii++ )
1571  {
1572  absorbance_integral[ ii ] = 0;
1573 
1574  const Scan* sc = &sl[ ii ];
1575  int value_count = sc->rvalues.size();
1576 
1577  double delta_r = sc->delta_r;
1578 
1579  // Integrate using trapezoid rule
1580  for ( int jj = 1; jj < value_count; jj++ )
1581  {
1582  double avg = ( sc->rvalues[ jj ] + sc->rvalues[ jj - 1 ] ) / 2.0;
1583  absorbance_integral[ ii ] += avg * delta_r;
1584  }
1585  }
1586 
1587  for ( int ii = 0; ii < scan_count; ii++ )
1588  scan_time[ ii ] = sl[ ii ].seconds;
1589 
1590  US_Matrix::lsfit( coeffs, scan_time, absorbance_integral, scan_count, order );
1591 
1592  QList< double > residuals;
1593 
1594  for ( int ii = 0; ii < scan_count; ii++ )
1595  {
1596  double fit = 0.0;
1597 
1598  for ( int jj = 0; jj < order; jj++ )
1599  fit += coeffs[ jj ] * pow( sl[ ii ].seconds, jj );
1600 
1601  residuals << absorbance_integral[ ii ] - fit;
1602  }
1603 
1604  return residuals;
1605 }
1606 
1607 // Return index of X (radius or wavelength) value
1608 int US_DataIO::index( US_DataIO::RawData* rdata, double xvalue )
1609 {
1610  return index( rdata->xvalues, xvalue );
1611 }
1612 
1613 // Return index of X (radius or wavelength) value
1614 int US_DataIO::index( US_DataIO::EditedData* edata, double xvalue )
1615 {
1616  return index( edata->xvalues, xvalue );
1617 }
1618 
1619 // Return index of X (radius or wavelength) value
1620 int US_DataIO::index( const QVector< double >& xvals, double xvalue )
1621 {
1622  int npoint = xvals.size();
1623  int last = npoint - 1;
1624 
1625  // If xvalue beyond values at extremes, return 1st or last index
1626  if ( xvalue <= xvals[ 0 ] ) return 0; // Return first index
1627 
1628  if ( xvalue >= xvals[ last ] ) return last; // Return last index
1629 
1630  // Otherwise, return the index to a match or the least difference
1631  double rdmin = qAbs( xvalue - xvals[ 0 ] );
1632  int mindx = 0;
1633 
1634  for ( int ii = 1; ii < npoint; ii++ )
1635  {
1636  double rdiff = qAbs( xvalue - xvals[ ii ] );
1637 
1638  if ( rdiff == 0.0 ) return ii; // Return match index
1639 
1640  if ( rdiff < rdmin )
1641  { // Save the minimum difference value; and its index for possible use
1642  rdmin = rdiff;
1643  mindx = ii;
1644  }
1645  }
1646 
1647  return mindx; // Return min-diff index
1648 }
1649 
1650 // Compose and return an error string
1651 QString US_DataIO::errorString( int code )
1652 {
1653  switch ( code )
1654  {
1655  case OK
1656  : return QObject::tr( "The operation completed successully" );
1657  case CANTOPEN : return QObject::tr( "The file cannot be opened" );
1658  case BADCRC : return QObject::tr( "The file was corrupted" );
1659  case NOT_USDATA: return QObject::tr( "The file was not valid scan data" );
1660  case BADTYPE : return QObject::tr( "The filetype was not recognized" );
1661  case BADXML : return QObject::tr( "The XML file was invalid" );
1662  case NODATA : return QObject::tr( "No legacy data files suitable for import were found" );
1663  case BAD_VERSION
1664  : return QObject::tr( "The file version is not supported" );
1665  case NO_GUID_MATCH
1666  : return QObject::tr( "GUIDs in raw data and edit data do not match" );
1667  }
1668 
1669  return QObject::tr( "Unknown error code" );
1670 }