__author__ = "saeedamen"  # Saeed Amen

#
# Copyright 2016 Cuemacro
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on a "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
#


import copy
from findatapy.util import ConfigManager
from findatapy.util import DataConstants
from findatapy.market.ioengine import SpeedCache

import concurrent.futures

import json

constants = DataConstants()


# from deco import *

class Market(object):
    """Higher level class which fetches market data using underlying classes
    such as MarketDataGenerator.

    Also contains several other classes, which are for asset specific
    instances, for example for generating FX spot time series
    or FX volatility surfaces.
    """

    def __init__(self, market_data_generator=None, md_request=None):
        if market_data_generator is None:
            if constants.default_market_data_generator\
                    == "marketdatagenerator":
                from findatapy.market import MarketDataGenerator
                market_data_generator = MarketDataGenerator()
            elif constants.default_market_data_generator \
                    == 'cachedmarketdatagenerator':
                # NOT CURRENTLY IMPLEMENTED FOR FUTURE USE
                from finaddpy.market import CachedMarketDataGenerator
                market_data_generator = CachedMarketDataGenerator()
            else:
                from findatapy.market import MarketDataGenerator
                market_data_generator = MarketDataGenerator()

        self.speed_cache = SpeedCache()
        self._market_data_generator = market_data_generator
        self._filter = Filter()
        self._calculations = Calculations()
        self.md_request = md_request

    def fetch_market(self, md_request=None, md_request_df=None,
                     md_request_str=None, md_request_dict=None, tickers=None,
                     start_date=None, finish_date=None, best_match_only=False,
                     **kwargs):
        """Fetches market data for specific tickers

        The user does not need to know to the low level API for each data
        provider works. The MarketDataRequest needs to supply parameters that
        define each data request. It has details which include:
            ticker eg. EURUSD
            field eg. close
            category eg. fx
            data_source eg. bloomberg
            start_date eg. 01 Jan 2015
            finish_date eg. 01 Jan 2017

        It can also have many optional attributes, such as
            vendor_ticker eg. EURUSD Curncy
            vendor_field eg. PX_LAST

        We can also create MarketDataRequest objects, using strings
        eg. fx.quandl.daily.NYC.EURUSD (category.data_source.freq.cut.ticker),
        as Python dict, as DataFrame with various properties
        (like ticker, category etc.)

        Parameters
        ----------
        md_request : MarketDataRequest (or shorthand str/DataFrame/dict)
            Describing what market data to fetch

        md_request_df : DataFrame
            Another way to specify some of the data request properties

        md_request_str : str
            We can specify part of the MarketDataRequest as a string, which
            gets converted later (or a JSON object)

        Returns
        -------
        pd.DataFrame
            Contains the requested market data

        """

        if self.md_request is not None:
            md_request = self.md_request

        if isinstance(md_request, str):
            md_request_str = md_request
            md_request = MarketDataRequest()

        if isinstance(md_request, dict):
            md_request_dict = md_request
            md_request = MarketDataRequest()

        if isinstance(md_request, pd.DataFrame):
            md_request_df = md_request
            md_request = MarketDataRequest()

        # Any kwargs are assumed to be to set MarketDataRequest attributes
        if kwargs != {}:
            md_request = self._kwargs_to_md_request(kwargs, md_request)

        # When we have specified a string
        if md_request_str is not None:
            md_request = self.create_md_request_from_str(
                md_request_str, md_request=md_request,
                start_date=start_date, finish_date=finish_date,
                best_match_only=best_match_only, **kwargs)

            return self.fetch_market(md_request)

        # When we have specified predefined tickers
        if tickers is not None:
            md_request = self.create_md_request_from_tickers(
                tickers, md_request=md_request, start_date=start_date,
                finish_date=finish_date, best_match_only=best_match_only,
                **kwargs)

            return self.fetch_market(md_request)

        # When we have specified a DataFrame with tickers
        if md_request_df is not None:
            md_request = self.create_md_request_from_dataframe(
                md_request_df, md_request=md_request,
                start_date=start_date, finish_date=finish_date,
                best_match_only=best_match_only, **kwargs)

            return self.fetch_market(md_request)

        # Or directly as a string
        if md_request_str is not None:
            md_request = self.create_md_request_from_str(
                md_request, 
                start_date=start_date, finish_date=finish_date,
                best_match_only=best_match_only, **kwargs)

            return self.fetch_market(md_request)

        # Or directly as a dict
        if md_request_dict is not None:
            md_request = self.create_md_request_from_dict(
                md_request_dict, md_request=md_request,
                start_date=start_date, finish_date=finish_date)

            return self.fetch_market(md_request)

        # Or directly as a DataFrame
        if md_request_df is not None:
            md_request = self.create_md_request_from_dataframe(
                md_request_df, md_request=md_request,
                start_date=start_date, finish_date=finish_date,
                **kwargs)

            return self.fetch_market(md_request)

        data_frame = None

        if isinstance(md_request, list):
            md_request = self.flatten_list_of_lists(md_request)

            if len(md_request) == 1:
                md_request = md_request[0]

        # If we've got a list MarketDataRequest objects, use threading to 
        # independently call them
        if isinstance(md_request, list):
            if len(md_request) > 0:

                with concurrent.futures.ThreadPoolExecutor(
                        max_workers=md_request[0].list_threads) as executor:
                    df_list = list(executor.map(self.fetch_market, md_request))

                df_filtered_list = []

                for md, df in zip(md_request, df_list):

                    columns = []

                    for tick in md.tickers:
                        for fiel in md.fields:
                            columns.append(tick + "." + fiel)

                    if df is not None:
                        if df.empty:
                            df = pd.DataFrame(columns=columns)
                    else:
                        df = pd.DataFrame(columns=columns)

                    df_filtered_list.append(df)

                return self._calculations.join(df_filtered_list)
            else:
                return None

        key = md_request.generate_key()

        # If internet_load has been specified don't bother going to cache,
        # might end up calling lower level cache though through 
        # MarketDataGenerator)
        if "cache_algo" in md_request.cache_algo and md_request.push_to_cache:
            data_frame = self.speed_cache.get_dataframe(key)

        if data_frame is not None:
            return data_frame

        if md_request.split_request_chunks > 0:
            md_request_list = []

            if md_request.vendor_tickers is not None:

                tickers = copy.copy(md_request.tickers)
                vendor_tickers = copy.copy(md_request.vendor_tickers)

                for t, v in zip(tickers, vendor_tickers):
                    md = MarketDataRequest(md_request=md_request)
                    md.tickers = t
                    md.vendor_tickers = v
                    md.split_request_chunks = 0

                    md_request_list.append(md)

            return self.fetch_market(md_request_list)

        # Special cases when a predefined category has been asked
        if md_request.category is not None:

            if (
                    md_request.category == 'fx-spot-volume' and 
                    md_request.data_source == 'quandl'):
                # NOT CURRENTLY IMPLEMENTED FOR FUTURE USE
                from findatapy.market.fxclsvolume import FXCLSVolume
                fxcls = FXCLSVolume(
                    market_data_generator=self._market_data_generator)

                data_frame = fxcls.get_fx_volume(
                    md_request.start_date, md_request.finish_date,
                    md_request.tickers, cut="LOC",
                    data_source="quandl",
                    cache_algo=md_request.cache_algo)

            # For FX we have special methods for returning cross rates or total 
            # returns
            if md_request.category in ["fx", "fx-tot",
                                       "fx-tot-forwards"] \
                    and md_request.tickers is not None and \
                    md_request.abstract_curve is None:
                fxcf = FXCrossFactory(
                    market_data_generator=self._market_data_generator)

                if md_request.category == "fx":
                    type = "spot"
                elif md_request.category == "fx-tot":
                    type = "tot"

                elif md_request.category == "fx-tot-forwards":
                    type = "tot-forwards"

                if md_request.freq != "tick" and (md_request.fields == [
                    "close"] or md_request.fields == ["open"]) or \
                        (md_request.freq == "tick"
                         and md_request.data_source in ["dukascopy", "fxcm"]):
                    data_frame = fxcf.get_fx_cross(
                        md_request.start_date,
                        md_request.finish_date,
                        md_request.tickers,
                        cut=md_request.cut,
                        data_source=md_request.data_source,
                        freq=md_request.freq,
                        cache_algo=md_request.cache_algo,
                        type=type,
                        environment=md_request.environment,
                        fields=md_request.fields,
                        data_engine=md_request.data_engine,
                        md_request_default=md_request)

            # For FX implied volatility we can return the full surface
            if md_request.category == "fx-implied-vol":
                if md_request.tickers is not None \
                        and md_request.freq == "daily":
                    df = []

                    fxvf = FXVolFactory(
                        market_data_generator=self._market_data_generator)

                    for t in md_request.tickers:
                        if len(t) == 6:
                            df.append(
                                fxvf.get_fx_implied_vol(
                                    md_request.start_date,
                                    md_request.finish_date,
                                    t,
                                    md_request.fx_vol_tenor,
                                    cut=md_request.cut,
                                    data_source=md_request.data_source,
                                    part=md_request.fx_vol_part,
                                    cache_algo=md_request.cache_algo,
                                    environment=md_request.environment,
                                    field=md_request.fields,
                                    data_engine=md_request.data_engine,
                                    md_request_default=md_request))

                    if df != []:
                        data_frame = Calculations().join(df, how="outer")

            # For FX vol market return all the market data necessary for 
            # pricing options which includes FX spot, volatility surface, 
            # forward points, deposit rates
            if md_request.category == "fx-vol-market":
                if md_request.tickers is not None:
                    df = []

                    fxcf = FXCrossFactory(
                        market_data_generator=self._market_data_generator)
                    fxvf = FXVolFactory(
                        market_data_generator=self._market_data_generator)
                    rates = RatesFactory(
                        market_data_generator=self._market_data_generator)

                    # For each FX cross fetch the spot, vol and forward points
                    for t in md_request.tickers:
                        if len(t) == 6:
                            # Spot
                            df.append(
                                fxcf.get_fx_cross(
                                    start=md_request.start_date,
                                    end=md_request.finish_date,
                                    cross=t,
                                    cut=md_request.cut,
                                    data_source=md_request.data_source,
                                    freq=md_request.freq,
                                    cache_algo=md_request.cache_algo,
                                    type='spot',
                                    environment=md_request.environment,
                                    fields=md_request.fields,
                                    data_engine=md_request.data_engine,
                                    md_request_default=md_request))

                            # Entire FX vol surface
                            df.append(
                                fxvf.get_fx_implied_vol(
                                    md_request.start_date,
                                    md_request.finish_date,
                                    t,
                                    md_request.fx_vol_tenor,
                                    cut=md_request.cut,
                                    data_source=md_request.data_source,
                                    part=md_request.fx_vol_part,
                                    cache_algo=md_request.cache_algo,
                                    environment=md_request.environment,
                                    field=md_request.fields,
                                    data_engine=md_request.data_engine,
                                    md_request_default=md_request))

                            # FX forward points for every point on curve
                            df.append(rates.get_fx_forward_points(
                                md_request.start_date, md_request.finish_date,
                                t,
                                md_request.fx_forwards_tenor,
                                cut=md_request.cut,
                                data_source=md_request.data_source,
                                environment=md_request.environment,
                                cache_algo=md_request.cache_algo,
                                field=md_request.fields,
                                data_engine=md_request.data_engine,
                                md_request_default=md_request))

                    # Lastly fetch the base depos
                    df.append(rates.get_base_depos(
                        md_request.start_date,
                        md_request.finish_date,
                        self._get_base_depo_currencies(
                           md_request.tickers),
                        md_request.base_depos_tenor,
                        environment=md_request.environment,
                        cut=md_request.cut,
                        data_source=md_request.data_source,
                        cache_algo=md_request.cache_algo,
                        field=md_request.fields,
                        data_engine=md_request.data_engine,
                        md_request_default=md_request))

                    if df != []:
                        data_frame = Calculations().join(df, how='outer')

            if md_request.category == "fx-forwards-market":
                if md_request.tickers is not None:
                    df = []

                    fxcf = FXCrossFactory(
                        market_data_generator=self._market_data_generator)
                    rates = RatesFactory(
                        market_data_generator=self._market_data_generator)

                    # For each FX cross fetch the spot and forward points
                    for t in md_request.tickers:
                        if len(t) == 6:
                            # Spot
                            df.append(
                                fxcf.get_fx_cross(
                                    start=md_request.start_date,
                                    end=md_request.finish_date,
                                    cross=t,
                                    cut=md_request.cut,
                                    data_source=md_request.data_source,
                                    freq=md_request.freq,
                                    cache_algo=md_request.cache_algo,
                                    type='spot',
                                    environment=md_request.environment,
                                    fields=md_request.fields,
                                    data_engine=md_request.data_engine,
                                    md_request_default=md_request))

                            # FX forward points for every point on curve
                            df.append(rates.get_fx_forward_points(
                                md_request.start_date, md_request.finish_date,
                                t,
                                md_request.fx_forwards_tenor,
                                cut=md_request.cut,
                                data_source=md_request.data_source,
                                environment=md_request.environment,
                                cache_algo=md_request.cache_algo,
                                field=md_request.fields,
                                data_engine=md_request.data_engine,
                                md_request_default=md_request))

                    # Lastly fetch the base depos
                    df.append(rates.get_base_depos(
                        md_request.start_date,
                        md_request.finish_date,
                        self._get_base_depo_currencies(
                           md_request.tickers),
                        md_request.base_depos_tenor,
                        cut=md_request.cut,
                        data_source=md_request.data_source,
                        cache_algo=md_request.cache_algo,
                        environment=md_request.environment,
                        field=md_request.fields,
                        data_engine=md_request.data_engine,
                        md_request_default=md_request))

                    if df:
                        data_frame = Calculations().join(df, how='outer')

            # eg. for calculating total return indices from first principles (
            # rather than downloading them from a data vendor)
            if md_request.abstract_curve is not None:
                data_frame = md_request.abstract_curve.fetch_continuous_time_series \
                    (md_request, self._market_data_generator)

            if md_request.category == "crypto":
                # Add more features later
                data_frame = self._market_data_generator.fetch_market_data(
                    md_request)

            # TODO add more special examples here for different asset classes
            # the idea is that we do all the market data downloading here,
            # rather than elsewhere

        # By default: pass the market data request to MarketDataGenerator
        # if data_frame is not None:
        #    data_frame = None

        if data_frame is None:
            data_frame = self._market_data_generator.fetch_market_data(
                md_request)

        # Special case where we can sometimes have duplicated data times
        if md_request.freq == "intraday" and md_request.cut == "BSTP":
            data_frame = self._filter.remove_duplicate_indices(data_frame)

        # Push into cache
        if md_request.push_to_cache:
            if data_frame is not None:
                self.speed_cache.put_dataframe(key, data_frame)

        return data_frame

    def create_md_request_from_dataframe(self, md_request_df, md_request=None,
                                         start_date=None, finish_date=None,
                                         smart_group=True,
                                         keep_initial_md_request_att_cols=[
                                             "tickers", "fields", "freq"],
                                         **kwargs):

        md_list = []

        # Aggregate/shrink dataframe grouping it by common attributes
        # tickers, vendor_tickers, fields, vendor_fields
        if smart_group:
            md_request_df = ConfigManager().get_instance()\
                .smart_group_dataframe_tickers(
                md_request_df, ret_fields=md_request_df.columns.tolist())

        # Now populate MarketDataRequests based on the DataFrame
        for index, row in md_request_df.iterrows():
            if md_request is None:
                md_request_copy = MarketDataRequest()
            else:
                md_request_copy = MarketDataRequest(md_request=md_request)

            for col, val in row.items():

                try:
                    if "," in val:
                        val = val.split(",")
                except:
                    pass

                # Only override the initial setting in md_request if they are
                # None, or the user has specified this
                if getattr(md_request_copy,
                           col) is None or col not \
                        in keep_initial_md_request_att_cols:
                    try:
                        if isinstance(val, list):
                            val = self.flatten_list_of_lists(val)

                        getattr(type(md_request_copy), col).fset(
                            md_request_copy, val)
                    except:
                        pass

            if start_date is not None: 
                md_request_copy.start_date = start_date
                
            if finish_date is not None: 
                md_request_copy.finish_date = finish_date

            md_request_copy = self._kwargs_to_md_request(kwargs,
                                                         md_request_copy)

            md_list.append(md_request_copy)

        return md_list

    def create_md_request_from_dict(self, md_request_dict, md_request=None,
                                    start_date=None, finish_date=None,
                                    **kwargs):

        if md_request is None:
            md_request = MarketDataRequest()

        for k in md_request_dict.keys():
            getattr(type(md_request), k).fset(md_request, md_request_dict[k])

        if start_date is not None: md_request.start_date = start_date
        if finish_date is not None: md_request.finish_date = finish_date

        md_request = self._kwargs_to_md_request(kwargs, md_request)

        return md_request

    def create_md_request_from_tickers(self, tickers, md_request=None,
                                       start_date=None, finish_date=None,
                                       best_match_only=False, smart_group=True,
                                       **kwargs):
        md_request_list = []

        if isinstance(tickers, str):
            tickers = [tickers]

        for t in tickers:
            md_request_copy = self.create_md_request_from_str(
                '_.' + t, md_request=md_request, start_date=start_date,
                finish_date=finish_date, best_match_only=best_match_only,
                smart_group=smart_group)

            md_request_copy = self._kwargs_to_md_request(kwargs,
                                                         md_request_copy)
            md_request_list.append(md_request_copy)

        return self.flatten_list_of_lists(md_request_list)

    def _kwargs_to_md_request(self, kw, md_request):

        if not (isinstance(md_request, list)):
            # Any kwargs are assumed to be to set MarketDataRequest attributes
            if kw != {}:

                if md_request is None:
                    md_request = MarketDataRequest()

                for k in kw.keys():
                    setattr(md_request, k, kw[k])

            return md_request

        # Any kwargs are assumed to be to set MarketDataRequest attributes
        if kw != {}:
            md_request_mod_list = []

            for md in md_request:
                if md is None:
                    md = MarketDataRequest()

                for k in kw.keys():
                    setattr(md, k, kw[k])

                md_request_mod_list.append(md)

            md_request = md_request_mod_list

        return md_request

    def create_md_request_from_str(self, md_request_str, md_request=None,
                                   start_date=None, finish_date=None,
                                   best_match_only=False,
                                   smart_group=True, **kwargs):

        json_md_request = None

        # Try to parse str as JSON if that fails, then try as a str
        try:
            json_md_request = json.loads(md_request_str)

            if md_request is None:
                md_request = MarketDataRequest()

            for k in json_md_request.keys():
                getattr(type(md_request), k).fset(md_request,
                                                  json_md_request[k])

            if start_date is not None: md_request.start_date = start_date
            if finish_date is not None: md_request.finish_date = finish_date

            return md_request
        except:
            pass

        # If we failed to parse as a JSON, let's try as string
        if json_md_request is None:

            # Split up ticker string
            md_request_params = ConfigManager.split_ticker_string(
                md_request_str)

            environment = md_request_params[0]

            # The user can omit the environment
            if environment in constants.possible_data_environment:
                i = 0

            # Otherwise, what if the user wants to specify each 
            # property manually?
            elif environment == "raw" or environment == "r":
                # Here the user can specify any tickers/fields etc. they want, 
                # they don't have to be predefined
                # eg. raw.data_source.bloomberg.tickers.EURUSD.vendor_tickers.EURUSD Curncy
                if md_request is None:
                    md_request = MarketDataRequest()

                freeform_md_request = {}

                for c in range(1, len(md_request_params), 2):
                    f = md_request_params[c + 1]

                    if "," in f:
                        f = f.split(",")

                    freeform_md_request[md_request_params[c]] = f

                md_request.freeform_md_request = freeform_md_request

                if start_date is not None: 
                    md_request.start_date = start_date
                    
                if finish_date is not None: 
                    md_request.finish_date = finish_date

                md_request = self.create_md_request_from_freeform(md_request)
                md_request = self._kwargs_to_md_request(kwargs, md_request)

                return md_request

            # Otherwise we do a partial match of predefined tickers
            elif environment == "_":
                # Try a heuristic/approximate match eg. _.quandl.fx
                md_request_df = ConfigManager().get_instance()\
                    .free_form_tickers_query(
                    md_request_params[1:],
                    best_match_only=best_match_only,
                    smart_group=smart_group)

                md_request = self.create_md_request_from_dataframe(
                    md_request_df,
                    md_request=md_request, start_date=start_date,
                    finish_date=finish_date)

                md_request = self._kwargs_to_md_request(kwargs,
                                                           md_request)

                # if best_match_only:
                return md_request


            else:
                i = -1
                environment = None

            # Otherwise the user has specified the MarketDataRequest str in 
            # the form
            # category.data_source.freq.cut.tickers.field = fx.bloomberg.daily.NYC.EURUSD.close
            category = md_request_params[i + 1]
            data_source = md_request_params[i + 2]

            cut = None
            freq = None
            tickers = None
            fields = None

            # The freq, cut, tickers, fields are optional (in which case 
            # defaults will be used)
            try:
                freq = md_request_params[i + 3]
            except:
                pass

            try:
                cut = md_request_params[i + 4]
            except:
                pass

            # We can have multiple tickers and fields separately by a ticker
            try:
                tickers = md_request_params[i + 5]

                if ',' in tickers:
                    tickers = tickers.split(',')
            except:
                pass

            try:
                fields = md_request_params[i + 6]

                if ',' in fields:
                    fields = fields.split(',')
            except:
                pass

            if md_request is None:
                md_request = MarketDataRequest(category=category,
                                               data_source=data_source)
            else:
                md_request.category = category
                md_request.data_source = data_source

            if environment is not None: md_request.environment = environment
            if start_date is not None: md_request.start_date = start_date
            if finish_date is not None: md_request.finish_date = finish_date
            if freq is not None: md_request.freq = freq
            if cut is not None: md_request.cut = cut
            if tickers is not None: md_request.tickers = tickers
            if fields is not None: md_request.fields = fields

        return md_request

    def create_md_request_from_freeform(self, md_request,
                                        freeform_md_request=None,
                                        return_df=False, **kwargs):

        if freeform_md_request is None:
            freeform_md_request = md_request.freeform_md_request

        if freeform_md_request is None:
            return md_request

        if isinstance(freeform_md_request, pd.DataFrame):
            pass
        else:
            if not (isinstance(freeform_md_request, list)):
                freeform_md_request = [freeform_md_request]

            df = pd.DataFrame.from_dict(freeform_md_request)

        group_columns = [x for x in df.columns if
                         x not in ["tickers", "vendor_tickers"]]

        # Group by everything but the tickers, vendor_tickers (which are 
        # concatenated) and make sure we remove any duplicated tickers
        if group_columns:
            if "tickers" in df.columns and "vendor_tickers" not in df.columns:
                df = df.groupby(group_columns, as_index=False).agg(
                    {"tickers": list})

                for i in range(0, len(df.index)):
