manipulating a .dat file and plotting cumulative data
<p>I want to plot a quantity from a tedious-to-look-at <code>.dat</code> file, the #time column in the file extends from 0s to 70s, but I need to take a closer look at data (Nuclear Energy, in this case) from 25s to 35s.</p>

<p>I was wondering if there is a way I can manipulate the time column and corresponding other columns to record and plot data only for the required time span. </p>

<p>I already have some code which does the job for me for 0-70s:</p>

<pre><code>import matplotlib
matplotlib.use('Agg')

import os
import numpy as np
import matplotlib.pyplot as plt
import string
import math



# reads from flash.dat
def getQuantity(folder, basename, varlist):

        # quantities[0] should contain only the quantities of varlist[0]        
        quantities =[]
        for i in range(len(varlist)):
                quantities.append([])

        with open(folder + "/" + basename + ".dat", 'r') as f: # same as f = open(...) but closes the file afterwards.

                for line in f:
                        if not ('#' or 'Inf')  in line: # the first line and restarting lines look like this.   

                                  for i in range(len(varlist)):
                                        if(varlist[i]==NUCLEAR_ENERGY and len(quantities[i])&gt;0):
                                                quantities[i].append(float(line.split()[varlist[i]])+quantities[i][-1])
                                        else:
                                                quantities[i].append(float(line.split()[varlist[i]]))


        return quantities
# end def getQuantity

#create plot
plt.figure(1)

TIME = 0

NUCLEAR_ENERGY = 18

labels = ["time", "Nuclear Energy"]


flashFolder1 = '/home/trina/Pictures' # should be the flash NOT the flash/object folder.
lab1 = '176'


filename = 'flash' # 'flash' for flash.dat
nHorizontal = 1 # number of Plots in Horizontal Direction. Vertical Direction is set by program.
outputFilename = 'QuantityPlots_Nuclear.png'

variables = [NUCLEAR_ENERGY]


#Adjustments to set the size
nVertical = math.ceil(float(len(variables))/nHorizontal)   # = 6 for 16 = len(variables) &amp; nHorizontal = 3.
F = plt.gcf()           #get figure
DPI = F.get_dpi()
DefaultSize = F.get_size_inches()
F.set_size_inches( DefaultSize[0]*nHorizontal, DefaultSize[1]*nVertical )       #build no of subplots in figure

variables.insert(0,TIME) # time as needed as well
data1 = getQuantity(flashFolder1, filename, variables)
time1 = np.array(data1[0])      #time is first column



for n in [n+1 for n in range(len(variables)-1)]: #starts at 1
        ax=plt.subplot(nVertical, nHorizontal, n)   #for example (6,3,0 to 15) inside loop for 16 variables
        if (min(data1[n])&lt;0.0 or abs((min(data1[n]))/(max(data1[n])))&gt;=1.e-2):
                plt.plot(time1, data1[n],label=lab1) #, label = labels[variables[n]])
                legend = ax.legend(loc='upper right', frameon=False)

        else:
                plt.semilogy(time1, data1[n],label=lab1) #, label = labels[variables[n]])
                legend = ax.legend(loc='upper right', frameon=False)

plt.savefig(outputFilename)
</code></pre>

<p>Here is the figure I can produce from this code:</p>

<p><a href="http://i.stack.imgur.com/23vmx.png" rel="nofollow"><img src="http://i.stack.imgur.com/23vmx.png" alt="enter image description here"></a></p>

<p>and for your convenience I am also sharing the <code>.dat</code> file:</p>

<p><a href="https://www.dropbox.com/s/w4jbxmln9e83355/flash.dat?dl=0" rel="nofollow">https://www.dropbox.com/s/w4jbxmln9e83355/flash.dat?dl=0</a></p>

<p>Your suggestions are most appreciated.</p>

<p><strong>UPDATE:</strong> plot cumulative nuclear energy:</p>

<pre><code>x = df.query('25 &lt;= time &lt;= 35').set_index('time')
x['cum_nucl_energy'] = x.Nuclear_Energy.cumsum()
x.cum_nucl_energy.plot(figsize=(12,10))
</code></pre>

<p><a href="http://i.stack.imgur.com/mhF6z.png" rel="nofollow"><img src="http://i.stack.imgur.com/mhF6z.png" alt="enter image description here"></a></p>

<p><strong>Old answer:</strong></p>

<p>Using Pandas module</p>

<pre><code>import pandas as pd
import matplotlib
import matplotlib.pyplot as plt

matplotlib.style.use('ggplot')

fn = r'D:\temp\.data\flash.dat'
df = pd.read_csv(fn, sep='\s+', usecols=[0, 18], header=None, skiprows=[0], na_values=['Infinity'])
df.columns=['time', 'Nuclear_Energy']
df.query('25 &lt;= time &lt;= 35').set_index('time').plot(figsize=(12,10))
plt.show()
plt.savefig('d:/temp/out.png')
</code></pre>

<p><strong>Result:</strong></p>

<p><a href="http://i.stack.imgur.com/uQkCt.png" rel="nofollow"><img src="http://i.stack.imgur.com/uQkCt.png" alt="enter image description here"></a></p>

<p>Explanation:</p>

<pre><code>In [43]: pd.options.display.max_rows
Out[43]: 50

In [44]: pd.options.display.max_rows = 12

In [45]: df
Out[45]:
               time  Nuclear_Energy
0      0.000000e+00    0.000000e+00
1      1.000000e-07   -4.750169e+29
2      2.200000e-07   -5.699325e+29
3      3.640000e-07   -6.838392e+29
4      5.368000e-07   -8.206028e+29
5      7.441600e-07   -9.837617e+29
...             ...             ...
10210  6.046702e+01    7.160630e+44
10211  6.047419e+01    7.038907e+44
10212  6.048137e+01    6.934600e+44
10213  6.048856e+01    6.847015e+44
10214  6.049577e+01    6.765220e+44
10215  6.050298e+01    6.661930e+44

[10216 rows x 2 columns]

In [46]: df.query('25 &lt;= time &lt;= 35')
Out[46]:
           time  Nuclear_Energy
4534  25.001663    1.559398e+43
4535  25.006781    1.567793e+43
4536  25.011900    1.575844e+43
4537  25.017021    1.583984e+43
4538  25.022141    1.592015e+43
4539  25.027259    1.600200e+43
...         ...             ...
6521  34.966427    8.181516e+41
6522  34.972926    8.538806e+41
6523  34.979425    8.913695e+41
6524  34.985925    9.304403e+41
6525  34.992429    9.731310e+41
6526  34.998941    1.019862e+42

[1993 rows x 2 columns]

In [47]: df.query('25 &lt;= time &lt;= 35').set_index('time')
Out[47]:
           Nuclear_Energy
time
25.001663    1.559398e+43
25.006781    1.567793e+43
25.011900    1.575844e+43
25.017021    1.583984e+43
25.022141    1.592015e+43
25.027259    1.600200e+43
...                   ...
34.966427    8.181516e+41
34.972926    8.538806e+41
34.979425    8.913695e+41
34.985925    9.304403e+41
34.992429    9.731310e+41
34.998941    1.019862e+42

[1993 rows x 1 columns]
</code></pre>

python numpy matplotlib dataframe data-manipulation