convert perf_log to YAML (not pursued to a satisfactory end)

dev-tools/ad-hoc/JWu/partial_history.txt

+# Release 1.2.0
+| date                | hostname   | sysinfo       | tr | total    | MultiLayer   | CylindersInD | RotatedPyram | CoreShell    | SquareLattic | RadialParaCr | HexParaCryst | SSCA         | Mesocrystal  | PolMagCyl    |
+| 2015-06-30 15:16:12 | jcnsopc126 | linuxx8664gcc | 0  | 20.120   | 1.686        | 1.156        | 1.947        | 0.974        | 2.814        | 1.177        | 2.055        | 3.835        | 1.831        | 2.644        |
+| 2015-06-30 15:16:37 | jcnsopc126 | linuxx8664gcc | 0  | 20.044   | 1.678        | 1.176        | 2.018        | 0.979        | 2.829        | 1.202        | 2.069        | 3.835        | 1.823        | 2.436        |
+| 2015-06-30 15:17:36 | jcnsopc126 | linuxx8664gcc | 0  | 20.347   | 1.699        | 1.319        | 2.010        | 1.003        | 2.822        | 1.197        | 2.031        | 3.861        | 1.890        | 2.515        |
+| 2015-06-30 15:19:12 | jcnsopc126 | linuxx8664gcc | 0  | 19.853   | 1.684        | 1.156        | 1.951        | 0.979        | 2.815        | 1.187        | 2.026        | 3.798        | 1.832        | 2.426        |
+
+# Release 1.3.0
+| date                | hostname   | sysinfo       | tr | total    | MultiLayer   | CylindersInD | RotatedPyram | CoreShell    | SquareLattic | RadialParaCr | HexParaCryst | SSCA         | Mesocrystal  | PolMagCyl    |
+| 2015-07-31 14:38:50 | jcnsopc63  | linuxx8664gcc | 0  | 15.590   | 1.537        | 0.977        | 1.673        | 0.855        | 2.524        | 0.996        | 1.874        | 1.505        | 1.581        | 2.068        |
+
+# after some development, why it is slower now not clear (there was minor OutputData refactoring
+| date                | hostname   | sysinfo       | tr | total    | MultiLayer   | CylindersInD | RotatedPyram | CoreShell    | SquareLattic | RadialParaCr | HexParaCryst | SSCA         | Mesocrystal  | PolMagCyl    |
+| 2015-09-23 14:54:52 | jcnsopc126 | linuxx8664gcc | 0  | 17.658   | 1.692        | 1.148        | 1.602        | 0.983        | 2.793        | 1.174        | 2.163        | 1.742        | 1.868        | 2.495        |
+
+# after masking
+| date                | hostname   | sysinfo       | tr | total    | MultiLayer   | CylindersInD | RotatedPyram | CoreShell    | SquareLattic | RadialParaCr | HexParaCryst | SSCA         | Mesocrystal  | PolMagCyl    |
+| 2015-09-28 17:19:17 | jcnsopc126 | linuxx8664gcc | 0  | 21.595   | 2.108        | 1.412        | 1.888        | 1.275        | 3.615        | 1.496        | 2.478        | 2.139        | 2.340        | 2.845 |
+
+# working on mask performance
+| date                | hostname   | sysinfo       | tr | total    | MultiLayer   | CylindersInD | RotatedPyram | CoreShell    | SquareLattic | RadialParaCr | HexParaCryst | SSCA         | Mesocrystal  | PolMagCyl    |
+| 2015-09-29 11:41:26 | jcnsopc126 | linuxx8664gcc | 0  | 18.188   | 1.793        | 1.172        | 1.647        | 1.012        | 2.945        | 1.290        | 2.197        | 1.798        | 1.849        | 2.486 |
+| 2015-09-29 11:46:58 | jcnsopc126 | linuxx8664gcc | 0  | 17.975   | 1.852        | 1.165        | 1.586        | 0.977        | 2.834        | 1.183        | 2.136        | 1.860        | 1.879        | 2.502 |
+
+# after RectangularDetector refactoring
+| date                | hostname   | sysinfo       | tr | total    | MultiLayer   | CylindersInD | RotatedPyram | CoreShell    | SquareLattic | RadialParaCr | HexParaCryst | SSCA         | Mesocrystal  | PolMagCyl    |
+| 2015-10-21 10:42:06 | jcnsopc126 | linuxx8664gcc | 0  | 19.284   | 1.781        | 1.333        | 1.785        | 1.133        | 3.145        | 1.404        | 2.156        | 1.847        | 1.884        | 2.818        |
+| 2015-10-21 10:42:30 | jcnsopc126 | linuxx8664gcc | 0  | 19.756   | 1.737        | 1.336        | 1.842        | 1.200        | 3.262        | 1.397        | 2.116        | 1.912        | 2.013        | 2.942|
+
+# PreRelease state. Seems that much worser. Simulation normalize?
+| date                | hostname   | sysinfo       | tr | total    | MultiLayer   | CylindersInD | RotatedPyram | CoreShell    | SquareLattic | RadialParaCr | HexParaCryst | SSCA         | Mesocrystal  | PolMagCyl    |
+| 2015-10-28 17:04:26 | jcnsopc126 | linuxx8664gcc | 0  | 25.693   | 2.573        | 1.693        | 2.410        | 1.472        | 4.134        | 1.748        | 2.856        | 2.376        | 2.715        | 3.716|
+
+# PreRelease. Moved checkPolarizationPresent() out of the loop.
+| date                | hostname   | sysinfo       | tr | total    | MultiLayer   | CylindersInD | RotatedPyram | CoreShell    | SquareLattic | RadialParaCr | HexParaCryst | SSCA         | Mesocrystal  | PolMagCyl    |
+| 2015-10-29 09:17:30 | jcnsopc126 | linuxx8664gcc | 0  | 20.586   | 2.386        | 1.166        | 1.773        | 0.890        | 3.449        | 1.195        | 2.570        | 1.687        | 2.452        | 3.020        |
+
+# PreRelease. If I try to backup mP_specular_info->getInCoefficients(alpha_i, 0.0, wavelength)); in InterferenceFunctionStrategy
+| date                | hostname   | sysinfo       | tr | total    | MultiLayer   | CylindersInD | RotatedPyram | CoreShell    | SquareLattic | RadialParaCr | HexParaCryst | SSCA         | Mesocrystal  | PolMagCyl    |
+| 2015-10-29 09:41:25 | jcnsopc126 | linuxx8664gcc | 0  | 19.921   | 2.416        | 0.984        | 1.654        | 0.778        | 3.333        | 1.027        | 2.923        | 1.505        | 2.359        | 2.942        |
+
+# PreRelease-1.5.0
+| date                | hostname   | sysinfo       | tr | total    | MultiLayer   | CylindersInD | RotatedPyram | CoreShell    | SquareLattic | RadialParaCr | HexParaCryst | SSCA         | Mesocrystal  | PolMagCyl    |
+| 2016-02-11 14:10:57 | jcnsopc126 | linuxx8664gcc | 0  | 18.247   | 1.981        | 0.920        | 1.418        | 0.696        | 2.578        | 0.950        | 3.899        | 1.381        | 2.008        | 2.417        |

dev-tools/ad-hoc/JWu/perf-log-old2yaml.py

+#!/usr/bin/env python3
+
+'''
+To be used only once: convert performance log from legacy format to YAML
+'''
+
+import datetime, re, sys, pytz, yaml
+from collections import OrderedDict
+from pytz import timezone
+
+################################################################################
+# Improved YAML dumper, to write dicts and lists in nice form
+################################################################################
+
+class FlowSeq( list ): pass
+
+# The improved YAML dumper
+
+def improved_dump(data, stream=None, Dumper=yaml.Dumper, **kwds):
+    class ImprovedDumper(Dumper):
+        pass
+    def odict_representer(dumper, data):
+        return dumper.represent_mapping(
+            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+            data.items())
+    def flowseq_representer(dumper, data):
+        return dumper.represent_sequence(
+            yaml.resolver.BaseResolver.DEFAULT_SEQUENCE_TAG,
+            data,
+            flow_style=True )
+    ImprovedDumper.add_representer(OrderedDict, odict_representer)
+    ImprovedDumper.add_representer(FlowSeq, flowseq_representer)
+    return yaml.dump(data, stream, ImprovedDumper,
+                     allow_unicode=True, encoding='utf-8',
+                     default_flow_style=False, indent=4, width=70, **kwds)
+
+# Just for documentation, not used here: the corresponding loader.
+
+def improved_load(stream, Loader=yaml.Loader, object_pairs_hook=OrderedDict):
+    class ImprovedLoader(Loader):
+        pass
+    def construct_mapping(loader, node):
+        loader.flatten_mapping(node)
+        return object_pairs_hook(loader.construct_pairs(node))
+    ImprovedLoader.add_constructor(
+        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+        construct_mapping)
+    return yaml.load(stream, ImprovedLoader)
+
+################################################################################
+# Time conversion
+################################################################################
+
+garching = timezone('Europe/Berlin')
+def nice_datetime( dt ):
+    rounded = dt - datetime.timedelta(microseconds=dt.microsecond)
+    return garching.localize(rounded).isoformat()
+
+################################################################################
+# Standarized blocks
+################################################################################
+
+def coder(value, offset_coder, offset, precision):
+    ret = OrderedDict()
+    ret['value'] = value
+    ret['offset_coder'] = offset_coder
+    ret['offset'] = offset
+    ret['precision'] = precision
+    return ret
+
+def scan(j):
+    ret = OrderedDict()
+    ret['scan_parameters'] = OrderedDict()
+    ret['scan_parameters']['omgs'] = 0.1*j
+    ret['time'] = 2.02
+    ret['monitor'] = random.randint(10000, 40000)
+    ret['image'] = "img%04i" % j
+    return ret
+
+################################################################################
+# Main
+################################################################################
+
+o = []
+
+fni = 'partial_history.txt'
+fno = 'performance_log2.yaml'
+fd = open( fni, 'r' )
+ti = fd.read()
+fd.close()
+
+aa = re.split( r'\n\s*?\n', ti )
+for a in aa: # block
+    bb = re.split( r'\n', a.rstrip() )
+    m = re.match( r'#\s*(.*)', bb[0] )
+    if not m:
+        raise "block header not found"
+    comment = m.group(1)
+    # print("### "+comment[0:20]+" --> %i" %( len(bb)-2 ) )
+    header = re.split( r'\s*\|\s*', bb[1] )[1:-1]
+    if header[0]!="date":
+        raise "unexpected block header"
+    nentry = len(header)
+    outcome = []
+    for b in bb[2:]:
+        c = re.split( r'\s*\|\s*', b )[1:-1]
+        # print("'"+b+"'")
+        if len(c)!=len(header):
+            raise "wrong number of records"
+        result = OrderedDict()
+        for i in range(len(header)):
+            out = c[i]
+            if header[i]=="tr":
+                out = int(out)
+            elif i>3:
+                out = float(out)
+            result[header[i]] = out
+        outcome.append( result )
+    entry = OrderedDict()
+    entry['version'] = comment
+    entry['outcome'] = outcome
+    o.append( entry )
+
+yaml_str = improved_dump( o )
+fd = open( fno, 'wb' )
+fd.write( yaml_str )
+fd.close()