NEW: change to kaitai lazy parsing of WDS (and images)

Raw Spectra data is loaded on-demand from files

Author: sem-geologist

lib/parsers/cameca.py

@@ -92,6 +92,29 @@ def __eq__(self, other):
 Cameca.ElementT = ElementT
 
 
+class LazyData(Cameca.LazyData):
+
+    def _read(self):
+        seek_to_pos = self.offset + self.size
+        self._root._io.seek(seek_to_pos)
+
+    @property
+    def lazy_bytes(self):
+        if not hasattr(self, "_m_lazy_bytes"):
+            io_fn = self._root._io._io.name
+            with open(io_fn, "rb") as fn:
+                fn.seek(self.offset)
+                self._m_lazy_bytes = fn.read(self.size)
+        return self._m_lazy_bytes
+
+    @lazy_bytes.deleter
+    def lazy_bytes(self):
+        del self._m_bytes
+
+
+Cameca.LazyData = LazyData
+
+
 class WdsScanSignal(Cameca.WdsScanSignal):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -110,7 +133,8 @@ def y_cps(self):
         if self._y_recalc_cps is not None:
             return self._y_recalc_cps
         if self._y_orig_cps is None:
-            self._y_orig_cps = np.frombuffer(self.data, dtype=np.float32)
+            self._y_orig_cps = np.frombuffer(self.data.lazy_bytes,
+                                             dtype=np.float32)
         return self._y_orig_cps
 
     @cached_property

lib/parsers/cameca_ks/cameca.ksy

@@ -644,7 +644,7 @@ types:
           (dataset_type != dataset_type::line_stage) and
           (dataset_type != dataset_type::line_beam)
       - id: data
-        size: frame_size
+        type: lazy_data(_root._io.pos, frame_size)
         repeat: expr
         repeat-expr: n_of_frames
         doc: |
@@ -793,7 +793,7 @@ types:
         type: f4
       - id: enabled
         type: u4
-        doc: use in calculations
+        doc: is this item used in calculations
       - id: peak_raw_cts
         type: s4
       - id: bkgd_1_raw_cts
@@ -808,6 +808,10 @@ types:
         type: s4
       - id: reserved_4
         size-eos: true
+    instances:
+      net_intensity:
+        value: (peak_cps - bkgd_inter_cps) / beam_current
+    -webide-representation: '{net_intensity:str}'
 
   wds_qti_signal:
     params:
@@ -912,7 +916,7 @@ types:
       - id: data_array_size
         type: u4
       - id: data
-        size: data_array_size
+        type: lazy_data(_root._io.pos, data_array_size)
       - id: not_re_flag
         type: u4
       - id: signal_name
@@ -1407,6 +1411,29 @@ types:
         type: qti_wds_measurement_setups
         if: wds_measurement_struct_type >= 19
 
+  lazy_data:
+    doc: |
+          its _read method needs to be reimplemented in target language
+          to seek in the stream by provided size by parameter instead
+          of reading into memory
+    params:
+      - id: offset
+        type: u4
+      - id: size
+        type: u4
+    instances:
+      lazy_bytes:
+        io: _root._io
+        pos: offset
+        size: size
+        doc: "lazily parsed bytes on-demand"
+    seq:
+      - id: parsed_bytes
+        size: size
+        doc: |
+          this needs to be get rid off in target language and replaced with
+          relative seek.
+  
   wds_scan_spect_setup:
     seq:
       - id: xtal
@@ -1568,7 +1595,7 @@ types:
       unix_timestamp:
         value: ms_filetime / 10000000. - 11644473600
         doc: 'seconds since Jan 1 1970'
-  
+
   element_t:
     seq:
       - id: atomic_number

lib/parsers/cameca_ks/cameca.py

@@ -625,7 +625,7 @@ def _read(self):
 
             self.data = [None] * (self.n_of_frames)
             for i in range(self.n_of_frames):
-                self.data[i] = self._io.read_bytes(self.frame_size)
+                self.data[i] = self._root.LazyData(self._root._io.pos(), self.frame_size, self._io, self, self._root)
 
             self.reserved_0 = self._io.read_bytes(56)
             self.lut_name = self._root.CSharpString(self._io, self, self._root)
@@ -680,6 +680,36 @@ def _read(self):
             self.user_comment = (self._io.read_bytes_full()).decode(u"CP1252")
 
 
+    class LazyData(KaitaiStruct):
+        """its _read method needs to be reimplemented in target language
+        to seek in the stream by provided size by parameter instead
+        of reading into memory
+        """
+        def __init__(self, offset, size, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self.offset = offset
+            self.size = size
+            self._read()
+
+        def _read(self):
+            self.parsed_bytes = self._io.read_bytes(self.size)
+
+        @property
+        def lazy_bytes(self):
+            """lazily parsed bytes on-demand."""
+            if hasattr(self, '_m_lazy_bytes'):
+                return self._m_lazy_bytes if hasattr(self, '_m_lazy_bytes') else None
+
+            io = self._root._io
+            _pos = io.pos()
+            io.seek(self.offset)
+            self._m_lazy_bytes = io.read_bytes(self.size)
+            io.seek(_pos)
+            return self._m_lazy_bytes if hasattr(self, '_m_lazy_bytes') else None
+
+
     class QtiDataItem(KaitaiStruct):
         def __init__(self, _io, _parent=None, _root=None):
             self._io = _io
@@ -1124,6 +1154,14 @@ def _read(self):
             self.reserved_3 = self._io.read_s4le()
             self.reserved_4 = self._io.read_bytes_full()
 
+        @property
+        def net_intensity(self):
+            if hasattr(self, '_m_net_intensity'):
+                return self._m_net_intensity if hasattr(self, '_m_net_intensity') else None
+
+            self._m_net_intensity = ((self.peak_cps - self.bkgd_inter_cps) / self.beam_current)
+            return self._m_net_intensity if hasattr(self, '_m_net_intensity') else None
+
 
     class XtalT(KaitaiStruct):
         def __init__(self, _io, _parent=None, _root=None):
@@ -1141,7 +1179,7 @@ def rev_name(self):
                 return self._m_rev_name if hasattr(self, '_m_rev_name') else None
 
             _pos = self._io.pos()
-            self._io.seek((0 if self.first_byte > 0x20 else 1))
+            self._io.seek((0 if self.first_byte > 32 else 1))
             self._m_rev_name = (self._io.read_bytes_full()).decode(u"CP1252")
             self._io.seek(_pos)
             return self._m_rev_name if hasattr(self, '_m_rev_name') else None
@@ -1517,7 +1555,7 @@ def _read(self):
             self.dwell_time = self._io.read_f4le()
             self.beam_size = self._io.read_u4le()
             self.data_array_size = self._io.read_u4le()
-            self.data = self._io.read_bytes(self.data_array_size)
+            self.data = self._root.LazyData(self._root._io.pos(), self.data_array_size, self._io, self, self._root)
             self.not_re_flag = self._io.read_u4le()
             self.signal_name = self._root.CSharpString(self._io, self, self._root)
             self.reserved_0 = self._io.read_bytes(4)