__author__ = "saeedamen"  # Saeed Amen

#
# Copyright 2016 Cuemacro
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on a "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
#

from typing import List
import io
import datetime
import json
from dateutil.parser import parse

import codecs
import glob
import shutil
import copy
import os.path

import math

import numpy as np
import pandas as pd

# For deprecated Arctic
try:
    from arctic import Arctic
    import pymongo
except:
    pass

# For ArcticDB
try:
    import arcticdb as adb
except:
    pass

# Needs this for AWS S3 bucket support
try:
    from s3fs import S3FileSystem
except:
    pass

# For Polars support
try:
    import polars as pl
except:
    pass

# pyarrow necessary for caching
try:
    import pyarrow as pa
except:
    pass

# For reading and writing to S3
try:
    import pyarrow.fs
    import pyarrow.parquet as pq

    from s3fs import S3FileSystem
except:
    pass

try:
    import redis
except:
    pass

# For accessing Excel
from openpyxl import load_workbook

from findatapy.util.dataconstants import DataConstants
from findatapy.util.loggermanager import LoggerManager

constants = DataConstants()

class IOEngine(object):
    """Write and reads time series data to disk in various formats, CSV, HDF5
    (fixed and table formats), MongoDB/Arctic and ArcticDB are supported.

    Can be used to save down output of finmarketpy backtests and also to cache
    market data locally.

    Also supports BColz (but not currently stable). Planning to add other
    interfaces such as SQL etc.
    """

    def __init__(self, cloud_credentials: dict = constants.cloud_credentials):
        self.cloud_credentials = self._convert_cred(cloud_credentials)

    @staticmethod


    # Functions to handle Excel on disk
    def write_time_series_to_excel(self,
                                   fname: str,
                                   sheet: str,
                                   data_frame: pd.DataFrame,
                                   create_new: bool = False):
        """Writes Pandas data frame to disk in Excel format

        Parameters
        ----------
        fname : str
            Excel filename to be written to
        sheet : str
            sheet in Excel
        data_frame : DataFrame
            data frame to be written
        create_new : boolean
            to create a new Excel file
        """

        if (create_new):
            writer = pd.ExcelWriter(fname, engine="xlsxwriter")
        else:
            if self.path_exists(fname):
                book = load_workbook(fname)
                writer = pd.ExcelWriter(fname, engine="xlsxwriter")
                writer.book = book
                writer.sheets = dict((ws.title, ws) for ws in book.worksheets)
            else:
                writer = pd.ExcelWriter(fname, engine="xlsxwriter")

        data_frame.to_excel(writer, sheet_name=sheet, engine="xlsxwriter")

        writer.save()
        writer.close()

    def write_time_series_to_excel_writer(self,
                                          writer,
                                          sheet: str,
                                          data_frame: pd.DataFrame):
        """Writes Pandas data frame to disk in Excel format for a writer

        Parameters
        ----------
        writer : ExcelWriter
            File handle to use for writing Excel file to disk
        sheet : str
            sheet in Excel
        data_frame : DataFrame
            data frame to be written
        """
        data_frame.to_excel(writer, sheet, engine="xlsxwriter")

    def read_excel_data_frame(self,
                              f_name: str,
                              excel_sheet: str,
                              freq: str,
                              cutoff: str = None,
                              dateparse: str= None,
                              postfix: str = ".close",
                              intraday_tz: str = "UTC") -> pd.DataFrame:
        """Reads Excel from disk into DataFrame

        Parameters
        ----------
        f_name : str
            Excel file path to read
        freq : str
            Frequency of data to read (intraday/daily etc)
        cutoff : DateTime (optional)
            end date to read up to
        dateparse : str (optional)
            date parser to use
        postfix : str (optional)
            postfix to add to each columns
        intraday_tz : str
            timezone of file if uses intraday data

        Returns
        -------
        DataFrame
        """

        return self.read_csv_data_frame(f_name, freq, cutoff=cutoff,
                                        dateparse=dateparse,
                                        postfix=postfix,
                                        intraday_tz=intraday_tz,
                                        excel_sheet=excel_sheet)

    def remove_time_series_cache_on_disk(self,
                                         fname: str,
                                         engine: str = "hdf5_fixed",
                                         db_server: str = constants.db_server,
                                         db_port: int = constants.db_port,
                                         timeout: int = 10,
                                         username: int = None,
                                         password: int = None):

        logger = LoggerManager().getLogger(__name__)

        if "hdf5" in engine:
            engine = "hdf5"

        if engine == "redis":

            fname = os.path.basename(fname).replace(".", "_")

            try:
                r = redis.StrictRedis(host=db_server, port=db_port, db=0,
                                      socket_timeout=timeout,
                                      socket_connect_timeout=timeout)

                if fname == "flush_all_keys":
                    r.flushall()
                else:
                    # Allow deletion of keys by pattern matching
                    matching_keys = r.keys("*" + fname)

                    if matching_keys:
                        # Use pipeline to speed up command
                        pipe = r.pipeline()

                        for key in matching_keys:
                            pipe.delete(key)

                        pipe.execute()

                    # r.delete(fname)

            except Exception as e:
                logger.warning(
                    f"Cannot delete non-existent key {fname} in Redis: {str(e)}")
        elif engine.startswith("arcticdb:"):
            arcticdb_conn_str = engine.replace("arcticdb:", "", 1)
            ac = adb.Arctic(arcticdb_conn_str)

            # fname = os.path.basename(fname).replace(".", "_")
            try:
                database = ac[fname]
            except:
                pass

            if database is None:
                ac.delete_library(fname)
                logger.info(f"Deleted ArcticDB library: {fname}")
            else:
                logger.info(f"Got ArcticDB library: {fname}")

        elif engine == "arctic":
            from arctic import Arctic
            import pymongo

            socketTimeoutMS = 30 * 1000
            fname = os.path.basename(fname).replace(".", "_")

            logger.info(f"Load MongoDB library: {fname}")

            if username is not None and password is not None:
                c = pymongo.MongoClient(
                    host=f"mongodb://{username}:{password}@{str(db_server)}:{str(db_port)}",
                    connect=False)  # , username=username, password=password)
            else:
                c = pymongo.MongoClient(
                    host=f"mongodb://{str(db_server)}:{str(db_port)}",
                    connect=False)

            store = Arctic(c, socketTimeoutMS=socketTimeoutMS,
                           serverSelectionTimeoutMS=socketTimeoutMS,
                           connectTimeoutMS=socketTimeoutMS)

            store.delete_library(fname)

            c.close()

            logger.info(f"Deleted MongoDB library: {fname}")
        elif engine == "hdf5":
            h5_filename = self.get_h5_filename(fname)

            # delete the old copy
            try:
                os.remove(h5_filename)
            except:
                pass

    @staticmethod
    def _populate_arcticdb_dict(arcticdb_dict: dict = None):
        default_arcticdb_dict = copy.deepcopy(DataConstants().arcticdb_dict)

        if arcticdb_dict is None:
            return default_arcticdb_dict
        else:
            for k in default_arcticdb_dict.keys():
                if k not in arcticdb_dict.keys():
                    arcticdb_dict[k] = default_arcticdb_dict[k]

        return arcticdb_dict

    @staticmethod
    def _filter_out_matching(data_frame: pd.DataFrame,
                             filter_out_matching: str = None):
        if filter_out_matching is not None:
            cols = data_frame.columns

            new_cols = []

            for col in cols:
                if filter_out_matching not in col:
                    new_cols.append(col)

            data_frame = data_frame[new_cols]

        return data_frame

    ### functions to handle HDF5 on disk, arctic etc.
    def write_time_series_cache_to_disk(
            self,
            fname: str = None,
            data_frame: pd.DataFrame = None,
            engine: str = "hdf5_fixed",
            append_data: bool = False,
            db_server: str = constants.db_server,
            db_port: int = constants.db_port,
            username: str = constants.db_username,
            password: str = constants.db_password,
            filter_out_matching: str = None,
            timeout: int = 10,
            use_cache_compression: bool = constants.use_cache_compression,
            parquet_compression: str = constants.parquet_compression,
            use_pyarrow_directly: bool = False,
            md_request=None,
            ticker: str = None,
            cloud_credentials: dict = None,
            arcticdb_dict: dict = None):
        """Writes Pandas data frame to disk as Parquet, HDF5 format or bcolz
        format, in Arctic or to Redis

        Note, that Redis uses pickle (you must make sure that your Redis
        instance is not accessible from unverified users, given you should not
        unpickle from unknown sources)

        Parameters
        ---------
        fname : str
            path of file
        data_frame : DataFrame
            data frame to be written to disk
        engine : str
            "hdf5_fixed" - use HDF5 fixed format, very quick, but cannot append to this
            "hdf5_table" - use HDF5 table format, slower but can append to
            "parquet" - use Parquet
            "arctic" - use deprecated Arctic/MongoDB database
            "arcticdb:conn_str" - use ArcticDB (on disk)
            "redis" - use Redis
        append_data : bool
            False - write a fresh copy of data on disk each time
            True - append data to disk
        db_server : str
            Database server for arctic (default: "127.0.0.1")
        timeout : int
            Number of seconds to do timeout
        """

        logger = LoggerManager().getLogger(__name__)

        if md_request is not None:
            if fname is None:
                fname = ""

            fname = self.path_join(fname, md_request.create_category_key(
                ticker=ticker))

            if md_request.arcticdb_dict is not None:
                arcticdb_dict = md_request.arcticdb_dict

        # Default HDF5 format
        hdf5_format = "fixed"

        if "hdf5" in engine:
            hdf5_format = engine.split("_")[1]
            engine = "hdf5"

        if engine == "redis":
            fname = os.path.basename(fname).replace(".", "_")

            # Will fail if Redis is not installed
            try:
                r = redis.StrictRedis(host=db_server, port=db_port, db=0,
                                      socket_timeout=timeout,
                                      socket_connect_timeout=timeout)

                ping = r.ping()

                # If Redis is alive, try pushing to it
                if ping:
                    if data_frame is not None:
                        if isinstance(data_frame, pd.DataFrame):
                            mem = data_frame.memory_usage(deep="deep").sum()
                            mem_float = round(float(mem) / (1024.0 * 1024.0),
                                              3)

                            if mem_float < 500:


                                if use_cache_compression:
                                    ser = io.BytesIO()
                                    data_frame.to_parquet(ser,
                                                         compression="gzip")
                                    ser.seek(0)

                                    r.set("comp_" + fname, ser.read())
                                else:
                                    ser = io.BytesIO()
                                    data_frame.to_parquet(ser)
                                    ser.seek(0)

                                    r.set(fname, ser.read())

                                logger.info("Pushed " + fname + " to Redis")
                            else:
                                logger.warn(
                                    "Did not push " + fname + " to Redis, given size")
                    else:
                        logger.info(
                            f"Object {fname} is empty, not pushed to Redis.")
                else:
                    logger.warning(
                        f"Did not push {fname} to Redis given not running")

            except Exception as e:
                fname_msg = fname

                if len(fname_msg) > 150:
                    fname_msg = fname_msg[:149] + "..."

                error_msg = str(e)

                if len(error_msg) > 150:
                    error_msg = error_msg[:149] + "..."

                logger.warning(
                    f"Could not push {fname_msg} to Redis: {error_msg}")

        elif engine.startswith("arcticdb:"):
            arcticdb_conn_str = engine.replace("arcticdb:", "", 1)
            ac = adb.Arctic(arcticdb_conn_str)

            arcticdb_dict = IOEngine._populate_arcticdb_dict(
                arcticdb_dict=arcticdb_dict)

            database = None

            try:
                database = ac[fname]
            except:
                pass

            if database is None:
                ac.create_library(fname)
                logger.info(f"Created ArcticDB library: {fname}")
            elif arcticdb_dict["force_create_library"]:
                ac.delete_library(fname)
                ac.create_library(fname)
                logger.info(f"Deleted old library and created ArcticDB library: {fname}")
            else:
                logger.info(f"Loading existing ArcticDB library: {fname}")

            # Access the library
            library = ac[fname]
            logger.info(f"Got ArcticDB library: {fname}")

            data_frame = IOEngine._filter_out_matching(
                data_frame, filter_out_matching=filter_out_matching)

            try:
                if arcticdb_dict["write_style"] == "write":
                    library.write(
                        fname, data_frame,
                        prune_previous_versions=arcticdb_dict["prune_previous_versions"])

                    logger.info(f"Written ArcticDB library: {fname}")
                elif arcticdb_dict["write_style"] == "append":
                    library.append(
                        fname, data_frame,
                        prune_previous_versions=arcticdb_dict["prune_previous_versions"])

                    logger.info(f"Appended ArcticDB library: {fname}")
                elif arcticdb_dict["write_style"] == "update":
                    library.update(
                        fname, data_frame,
                        prune_previous_versions=arcticdb_dict[
                            "prune_previous_versions"])

                    logger.info(f"Updated ArcticDB library: {fname}")


            except Exception as e:
                logger.warning(
                    f"Could not write to ArcticDB library: {fname} {str(e)}")

        elif engine == "arctic":
            socketTimeoutMS = 30 * 1000
            fname = os.path.basename(fname).replace(".", "_")

            logger.info(f"Load Arctic/MongoDB library: {fname}...")

            if username is not None and password is not None:
                c = pymongo.MongoClient(
                    host="mongodb://" + username + ":" + password + "@" + str(
                        db_server) + ":" + str(db_port),
                    connect=False)  # , username=username, password=password)
            else:
                c = pymongo.MongoClient(
                    host="mongodb://" + str(db_server) + ":" + str(db_port),
                    connect=False)

            store = Arctic(c, socketTimeoutMS=socketTimeoutMS,
                           serverSelectionTimeoutMS=socketTimeoutMS,
                           connectTimeoutMS=socketTimeoutMS)

            database = None

            try:
                database = store[fname]
            except:
                pass

            if database is None:
                store.initialize_library(fname, audit=False)
                logger.info("Created MongoDB library: " + fname)
            else:
                logger.info(f"Got MongoDB library: {fname}")

            # Access the library
            library = store[fname]

            if "intraday" in fname:
                data_frame = data_frame.astype("float32")

            data_frame = IOEngine._filter_out_matching(
               data_frame, filter_out_matching=filter_out_matching)

            # Problems with Arctic when writing timezone to disk sometimes,
            # so strip
            data_frame = data_frame.copy().tz_localize(None)

            try:
                # Can duplicate values if we have existing dates
                if append_data:
                    library.append(fname, data_frame)
                else:
                    library.write(fname, data_frame)

                c.close()
                logger.info(f"Written MongoDB library: {fname}")
            except Exception as e:
                logger.warning(
                    f"Could not write MongoDB library: {fname} {str(e)}")

        elif engine == "hdf5":
            h5_filename = self.get_h5_filename(fname)

            # Append data only works for HDF5 stored as tables (but this is
            # much slower than fixed format) removes duplicated entries at
            # the end
            if append_data:
                store = pd.HDFStore(h5_filename, format=hdf5_format,
                                    complib="zlib", complevel=9)

                if "intraday" in fname:
                    data_frame = data_frame.astype("float32")

                # get last row which matches and remove everything after that
                # (because append function does not check for duplicated rows)
                nrows = len(store["data"].index)
                last_point = data_frame.index[-1]

                i = nrows - 1

                while (i > 0):
                    read_index = \
                    store.select("data", start=i, stop=nrows).index[0]

                    if read_index <= last_point: break

                    i = i - 1

                # Remove rows at the end, which are duplicates of the
                # incoming time series
                store.remove(key="data", start=i, stop=nrows)
                store.put(key="data", value=data_frame, format=hdf5_format,
                          append=True)
                store.close()
            else:
                h5_filename_temp = self.get_h5_filename(fname + ".temp")

                # delete the old copy
                try:
                    os.remove(h5_filename_temp)
                except:
                    pass

                store = pd.HDFStore(h5_filename_temp, complib="zlib",
                                    complevel=9)

                if "intraday" in fname:
                    data_frame = data_frame.astype("float32")

                store.put(key="data", value=data_frame, format=hdf5_format)
                store.close()

                # delete the old copy
                try:
                    os.remove(h5_filename)
                except:
                    pass

                # once written to disk rename
                os.rename(h5_filename_temp, h5_filename)

            logger.info("Written HDF5: " + fname)

        elif engine == "parquet":
            if ".parquet" not in fname:
                if fname[-5:] != ".gzip":
                    fname = fname + ".parquet"

            self.to_parquet(data_frame, fname,
                            cloud_credentials=cloud_credentials,
                            parquet_compression=parquet_compression,
                            use_pyarrow_directly=use_pyarrow_directly)
            # data_frame.to_parquet(fname, compression=parquet_compression)

            logger.info(f"Written Parquet: {fname}")
        elif engine == "csv":
            if ".csv" not in fname:
                fname = fname + ".csv"

            data_frame.to_csv(fname)

            logger.info(f"Written CSV: {fname}")

    def get_h5_filename(self, fname: str):
        """Strips h5 off filename returning first portion of filename

        Parameters
        ----------
        fname : str
            h5 filename to strip

        Returns
        -------
        str
        """
        if fname[-3:] == ".h5":
            return fname

        return fname + ".h5"

    def get_bcolz_filename(self, fname: str):
        """Strips bcolz off filename returning first portion of filename

        Parameters
        ----------
        fname : str
            bcolz filename to strip

        Returns
        -------
        str
        """
        if fname[-6:] == ".bcolz":
            return fname

        return fname + ".bcolz"

    def write_r_compatible_hdf_dataframe(self,
                                         data_frame: pd.DataFrame,
                                         fname: str,
                                         fields: List[str]=None):
        """Write a DataFrame to disk in as an R compatible HDF5 file.

        Parameters
        ----------
        data_frame : DataFrame
            data frame to be written
        fname : str
            file path to be written
        fields : list(str)
            columns to be written
        """

        logger = LoggerManager().getLogger(__name__)

        fname_r = self.get_h5_filename(fname)

        logger.info(f"About to dump R binary HDF5 - {fname_r}")
        data_frame32 = data_frame.astype("float32")

        if fields is None:
            fields = data_frame32.columns.values

        # decompose date/time into individual fields (easier to pick up in R)
        data_frame32["Year"] = data_frame.index.year
        data_frame32["Month"] = data_frame.index.month
        data_frame32["Day"] = data_frame.index.day
        data_frame32["Hour"] = data_frame.index.hour
        data_frame32["Minute"] = data_frame.index.minute
        data_frame32["Second"] = data_frame.index.second
        data_frame32["Millisecond"] = data_frame.index.microsecond / 1000

        data_frame32 = data_frame32[
            ["Year", "Month", "Day", "Hour", "Minute", "Second",
             "Millisecond"] + fields]

        cols = data_frame32.columns

        store_export = pd.HDFStore(fname_r)
        store_export.put("df_for_r", data_frame32, data_columns=cols)
        store_export.close()

    def read_time_series_cache_from_disk(self, fname: str,
                                         engine: str ="hdf5",
                                         start_date: str = None,
                                         finish_date: str = None,
                                         columns: List[str] = None,
                                         db_server: str = constants.db_server,
                                         db_port: int = constants.db_port,
                                         username: str = constants.db_username,
                                         password: str = constants.db_password,
                                         arcticdb_dict: dict = None,
                                         as_of=None):
        """Reads time series cache from disk in either HDF5 or bcolz

        Parameters
        ----------
        fname : str (or list)
            file to be read from
        engine : str (optional)
            "hd5" - reads HDF5 files (default)
            "arctic" - reads from deprecated Arctic/MongoDB database
            "arcticdb" - reads from ArcticDB (on disk storage)
            "bcolz" - reads from bcolz file (not fully implemented)
            "parquet" - reads from Parquet
        start_date : str/datetime (optional)
            Start date
        finish_date : str/datetime (optional)
            Finish data
        db_server : str
            IP address of MongoDB (default "127.0.0.1")

        Returns
        -------
        DataFrame
        """

        logger = LoggerManager.getLogger(__name__)

        data_frame_list = []

        if not (isinstance(fname, list)):
            if "*" in fname:
                fname = glob.glob(fname)
            else:
                fname = [fname]

        for fname_single in fname:
            logger.debug(f"Reading {fname_single}..")

            if engine == "parquet" and ".gzip" not in fname_single \
                    and ".parquet" not in fname_single:
                fname_single = f"{fname_single}.parquet"

            if engine == "redis":
                fname_single = os.path.basename(fname_single).replace(".", "_")

                msg = None

                try:
                    r = redis.StrictRedis(host=db_server, port=db_port, db=0)

                    # is there a compressed key stored?)
                    k = r.keys("comp_" + fname_single)

                    # If so, then it means that we have stored it as a
                    # compressed object if have more than 1 element, take the
                    # last (which will be the latest to be added)
                    if len(k) >= 1:
                        k = k[-1].decode("utf-8")

                        msg = r.get(k)
                        msg = io.BytesIO(msg)
                        msg = pd.read_parquet(msg)
                    else:
                        msg = r.get(fname_single)
                        msg = pd.read_parquet(msg.read())

                except Exception as e:
                    logger.info(
                        f"Cache not existent for {fname_single} in Redis: {str(e)}")

                if msg is None:
                    data_frame = None
                else:
                    logger.info(f"Load Redis cache: {fname_single}")

                    data_frame = msg  # pd.read_msgpack(msg)

            elif engine.startswith("arcticdb:"):
                arcticdb_conn_str = engine.replace("arcticdb:", "", 1)
                ac = adb.Arctic(arcticdb_conn_str)

                arcticdb_dict = IOEngine._populate_arcticdb_dict(arcticdb_dict)

                # Access the library
                try:
                    library = ac[fname_single]

                    if arcticdb_dict["allow_on_disk_filter"]:
                        date_range = None

                        if start_date is not None and finish_date is not None:
                            date_range = (start_date, finish_date)

                        item = library.read(fname_single,
                                            as_of=as_of,
                                            date_range=date_range,
                                            columns=columns,
                                            query_builder=arcticdb_dict["query_builder"])

                        logger.info(f"Read {fname_single} as of {str(as_of)}, between {str(date_range)}")
                    else:
                        item = library.read(fname_single, as_of=as_of)

                        logger.info(f"Read {fname_single}")

                    data_frame = item.data

                except Exception as e:
                    logger.warning(
                        f"Library may not exist or another error: {fname_single} & message is {str(e)}")
                    data_frame = None

            elif engine == "arctic":
                socketTimeoutMS = 2 * 1000

                import pymongo
                from arctic import Arctic

                fname_single = os.path.basename(fname_single).replace(".", "_")

                logger.info(f"Load Arctic/MongoDB library: {fname_single}")

                if username is not None and password is not None:
                    c = pymongo.MongoClient(
                        host=f"mongodb://{username}:{password}@{str(db_server)}:{str(db_port)}",
                        connect=False)  # , username=username, password=password)
                else:
                    c = pymongo.MongoClient(
                        host=f"mongodb://{str(db_server)}:{str(db_port)}", connect=False)

                store = Arctic(c, socketTimeoutMS=socketTimeoutMS,
                               serverSelectionTimeoutMS=socketTimeoutMS)

                # Access the library
                try:
                    library = store[fname_single]

                    if start_date is None and finish_date is None:
                        item = library.read(fname_single)

                    else:
                        from arctic.date import DateRange
                        item = library.read(fname_single, date_range=DateRange(
                            start_date.replace(tzinfo=None),
                            finish_date.replace(tzinfo=None)))

                    c.close()

                    logger.info(f"Read {fname_single}")

                    data_frame = item.data

                except Exception as e:
                    logger.warning(
                        f"Library may not exist or another error: {fname_single} & message is {str(e)}")
                    data_frame = None

            elif self.path_exists(self.get_h5_filename(fname_single)):
                store = pd.HDFStore(self.get_h5_filename(fname_single))
                data_frame = store.select("data")

                if "intraday" in fname_single:
                    data_frame = data_frame.astype("float32")

                store.close()

            elif self.path_exists(fname_single) and ".csv" in fname_single:
                data_frame = pd.read_csv(fname_single, index_col=0, usecols=columns)

                data_frame.index = pd.to_datetime(data_frame.index)

            elif self.path_exists(fname_single):
                data_frame = self.read_parquet(fname_single)

                # Can be too restrictive selecting columns
                # data_frame = self.read_parquet(fname_single, columns=columns)
                # data_frame = pd.read_parquet(fname_single)

            data_frame_list.append(data_frame)

        if len(data_frame_list) == 0:
            return None

        if len(data_frame_list) == 1:
            return data_frame_list[0]

        return data_frame_list

    ### functions for CSV reading and writing
    def write_time_series_to_csv(self, csv_path, data_frame):
        data_frame.to_csv(csv_path)

    def read_csv_data_frame(self, f_name, freq, cutoff=None, dateparse=None,
                            postfix=".close", intraday_tz="UTC",
                            excel_sheet=None):
        """Reads CSV/Excel from disk into DataFrame

        Parameters
        ----------
        f_name : str
            CSV/Excel file path to read
        freq : str
            Frequency of data to read (intraday/daily etc)
        cutoff : DateTime (optional)
            end date to read up to
        dateparse : str (optional)
            date parser to use
        postfix : str (optional)
