import timeit;

import copy;

import constants;
import crData;

import icrfAnalyser;

from helpers import dataFormatHelpers, cacheHelpers, helpers, footbotAnalysisHelpers;
import pyCreeper.crData;
import visualisation;

import icrfHelper;

RECALC_REWARD_CACHE = False;
RECALC_ICR_CACHE = False;
RECALC_ICR_SECONDARY_CACHE = False;
RECALC_CLEANED_DATA = False;

INFO_GAIN_COMPRESSION_TIME_INTERVAL = 20;


#======================================================== COMPLETION TIME ================================

def getCompletionTimes(dataPath_, numRuns_, convertToMinutes_=True, convertToHours_=False, targetResourceCollected_=9999):
    """
    Return completion times data for N runs, as an array N long.
    :param dataPath_: data path, relative to constants.DATA_PATH, e.g. static_wsr500/NR5_FOR_scatter10-7000_S
    """

    results_completionTime = [-1 for i in range(numRuns_)];

    #-- check if data needs to be calculated
    needsToRecalculateReward = False;
    if (RECALC_REWARD_CACHE):
        #-- recalculation was requested from outside of this function
        needsToRecalculateReward = True;
    else:
        cachedData = cacheHelpers.loadCachedRewardTotals(dataPath_, numRuns_);
        if (cachedData):
            #-- cache exists, no need to recalculate
            needsToRecalculateReward = False;
            results_completionTime = cachedData;
        else:
            needsToRecalculateReward = True;

    #-- calculate data if neccessary
    if (needsToRecalculateReward):
        for runNo in range(numRuns_):
            if (constants.USING_FOOTBOTS_SIMULATION_RESULTS):
                runData = crData.fileToArray((dataFormatHelpers.getBaseDataPath()+"/{}/Run{}_global.txt").format(dataPath_, runNo),"\t", True);
                #-- obtain reward data from global.txt
                for t in range(len(runData)):
                    if (runData[t][1] >= 100 or t == len(runData)-1):
                        results_completionTime[runNo] = runData[t][0];
                        #print("reward found at t=" + str(latestRewardReceivedEventTime));
                        break;
            else:
                #-- e-pucks, correct data only in infoEvents.txt
                runData = crData.fileToArray((dataFormatHelpers.getBaseDataPath()+"/{}/run{}_infoEvents.txt").format(dataPath_, runNo),"\t", True);
                rewardReceived = 0;
                for i in range(len(runData)):
                    eventType = runData[i][1];
                    time = runData[i][0];

                    #print(eventType,dataFormatHelpers.getEventDescriptionForType(constants.E_REWARD_RECEIVED) );
                    if (eventType == dataFormatHelpers.getEventDescriptionForType(constants.E_REWARD_RECEIVED)):
                        rewardReceived += 1;
                        if (rewardReceived <= targetResourceCollected_ and time > results_completionTime[runNo]):
                            #print("time found: " + str(time));
                            results_completionTime[runNo] = time;

            #print("max time: " + str(results_completionTimes[runNo]))

        #-- cache data
        cacheHelpers.cacheRewardTotals(results_completionTime, dataPath_, numRuns_);

    #-- convert data as requested
    if (convertToHours_):
        for runNo in range(numRuns_):
            results_completionTime[runNo] = (results_completionTime[runNo] / 3600.0);  # converting seconds to hours
    elif (convertToMinutes_):
        for runNo in range(numRuns_):
            results_completionTime[runNo] = (results_completionTime[runNo] / 60.0);  # converting seconds to minutes

    return results_completionTime;


#======================================================== ICR TOTALS ================================

def getICRData(dataPath_, numOfRuns_, numOfRobots_, maxTime_, rewardGainRate_):
    """
    :param dataPath_: data path, relative to constants.DATA_PATH, e.g. static_wsr500/NR5_FOR_scatter10-7000_S
    :param numOfRuns_:
    :param numOfRobots_:
    :param maxTime_:
    :param rewardGainRate_:
    :return: 2D array where 0th dimension is type of data (e.g. certain cost) and 1st dimension is values in different runs. The different returned types of data are:
        infoGainRate, uncertaintyCostTotal, displacementCostTotal, displacementCostCoeffMedian, misinformationCostTotal
    """
    results_infoGainRate = [];
    results_uncertaintyCostTotal = [];
    results_displacementCostTotal = [];
    results_displacementCostCoeffMedian = [];
    results_misinformationCostTotal = [];

    results_uncertaintyCostPerMinute = [];
    results_displacementCostPerMinute = [];
    results_misinformationCostPerMinute = [];

    #-- check if data needs to be calculated
    needsToRecalculate = True;
    if (RECALC_ICR_CACHE):
        #-- recalculation was requested from outside of this function
        pass
    else:
        cachedData = cacheHelpers.loadCachedICRTotals(dataPath_, numOfRuns_, INFO_GAIN_COMPRESSION_TIME_INTERVAL);
        if (cachedData):
            #-- cache exists, no need to recalculate
            needsToRecalculate = False;
            results_infoGainRate = cachedData[0];
            results_uncertaintyCostTotal = cachedData[1];
            results_displacementCostTotal = cachedData[2];
            results_displacementCostCoeffMedian = cachedData[3];
            results_misinformationCostTotal = cachedData[4];

    #-- calculate data
    completionTimes = getCompletionTimes(dataPath_, numOfRuns_, convertToMinutes_=True);

    returnData = [];
    for runNo in range(numOfRuns_):
        #-- get ICR data
        if (needsToRecalculate):
            #-- calculate data over time for the specific run
            icrDataOverTime = getRunICRDataOverTime(dataPath_, runNo, numOfRobots_, maxTime_, rewardGainRate_);

            #-- calculate cost totals by summing values in respective returned arrays
            results_uncertaintyCostTotal.append(sum(icrDataOverTime[5]));
            results_displacementCostTotal.append(sum(icrDataOverTime[6]));
            results_misinformationCostTotal.append(sum(icrDataOverTime[8]));

            #-- calculate medians
            results_displacementCostCoeffMedian.append(pyCreeper.crData.getMedianOfAList(icrDataOverTime[7], ignoreZeros_=True));

            #-- special calculations
            results_infoGainRate.append(icrfAnalyser.calculateInformationGainRate(icrDataOverTime[3],INFO_GAIN_COMPRESSION_TIME_INTERVAL));

            #print("max time: " + str(results_completionTimes[runNo]))


        #-- get costs per minute
        results_uncertaintyCostPerMinute.append(results_uncertaintyCostTotal[runNo] / completionTimes[runNo]);
        results_misinformationCostPerMinute.append(results_misinformationCostTotal[runNo] / completionTimes[runNo]);
        results_displacementCostPerMinute.append(results_displacementCostTotal[runNo] / completionTimes[runNo]);
        #print(" run time {}  C_U total {}  C_U per minute {}".format(completionTimes[runNo], results_uncertaintyCostTotal[runNo], results_uncertaintyCostPerMinute[runNo]));

    returnData = [
        results_infoGainRate, results_uncertaintyCostTotal, results_displacementCostTotal, results_displacementCostCoeffMedian, results_misinformationCostTotal,
        results_uncertaintyCostPerMinute, results_displacementCostPerMinute, results_misinformationCostPerMinute];

    #-- cache data
    cacheHelpers.cacheICRTotals(returnData, dataPath_, numOfRuns_, INFO_GAIN_COMPRESSION_TIME_INTERVAL);

    return returnData;

#======================================================== ICR OVER TIME ================================

def getICRDataOverTime(dataPath_, numOfRuns_, numOfRobots_, maxTime_, rewardGainRate_):
    """
    :param dataPath_:
    :param numOfRuns_:
    :param numOfRobots_:
    :param maxTime_:
    :param rewardGainRate_:
    :return: 3D array where 0th dimension represents different runs and the other two dimensions follow the format of getRunICRDataOverTime
    """
    returnData = [];
    for runNo in range(numOfRuns_):
        returnData.append(getRunICRDataOverTime(dataPath_, runNo, numOfRobots_, maxTime_, rewardGainRate_));
    return returnData;


#======================================================== ICR SECONDARY DATA ================================

def getICRSecondaryData(dataPath_, numOfRuns_, numOfRobots_, maxTime_):
    """
    :param dataPath_: data path, relative to constants.DATA_PATH, e.g. static_wsr500/NR5_FOR_scatter10-7000_S
    :param numOfRuns_:
    :param numOfRobots_:
    :param maxTime_:
    :param rewardGainRate_:
    :return: 2D array where 0th dimension is type of data (e.g. certain cost) and 1st dimension is values in different runs. The different returned types of data are:
        unemploymentRate, unevenEmploymentRate, foragingTripTravelTime, results_averageWorksiteDiscoveryTime (only the 1st discovery of each worksite taken into account)
    """
    results_unemploymentRate = [];
    results_unevenEmploymentRate = [];
    results_foragingTripTravelTime = [];
    results_averageWorksiteDiscoveryTime = [];

    #-- check if data needs to be calculated
    if (RECALC_ICR_SECONDARY_CACHE):
        #-- recalculation was requested from outside of this function
        pass
    else:
        cachedData = cacheHelpers.loadCachedICRSecondaryTotals(dataPath_, numOfRuns_);
        if (cachedData):
            #-- cache exists, no need to recalculate
            return cachedData;

    #-- calculate data if neccessary
    returnData = [];

    for runNo in range(numOfRuns_):

        if (constants.USING_FOOTBOTS_SIMULATION_RESULTS):
            runDataPathPrefix = "{}/Run{}".format(dataPath_, runNo);
        else:
            #-- e-pucks, data can be calculated
            #-- calculate data over time for the specific run
            runDataPathPrefix = "{}/run{}".format(dataPath_, runNo);


        #-- get over time site / robot data
        filePath = (dataFormatHelpers.getBaseDataPath()+"/{}_infoEvents_cleaned.txt").format(runDataPathPrefix);
        (sitesInitialVolumes, sitesInitialValues,                        #0-1
            numOfSites, totalEnvReward, actualRewardData,                       #2-4
            sitesCompleted,                                                 #5
            sitesSubscribedRobots, sitesSubscribedRobotsByRecruitment,      #6-7
            sitesReachedRobots, sitesLadenRobots, sitesEarningRobots,
            brokenRobotIds, robotIds) = icrfAnalyser.GET_ROBOT_WORKSITE_STATE_DATA_FUNCTION(filePath,maxTime_, False);

        #-- debug setup
        robotIdIndexForForagingTripTimeDebug = 10; #use a number or -1 for any robot, > numOfRobots for no robot

        #-- process the data over time
        maxTime = len(sitesSubscribedRobots);
        unemploymentRatesInRun = [];
        differencePercentageOfSubscribedRobotsFromIdealInRun = [];
        robotsForagingTripStartTimes = [0 for robotId in range(len(robotIds))];
        robotsForagingTripTimes = [];
        worksiteDiscoveryTimes = [9999 for worksiteId in range(numOfSites[0])];


        for t in range(maxTime):
            totalNumOfSubscribedRobots = 0.0;
            isRunCompleted = len(sitesCompleted[t]) == numOfSites[t];

            numOfWorkingRobots = numOfRobots_-len(brokenRobotIds[t]);


            if (not isRunCompleted):
                #-- first pass through worksites, get totals etc.
                for w in range(len(sitesSubscribedRobots[t])):
                    numOfSubscribedRobots = len(sitesSubscribedRobots[t][w]);
                    totalNumOfSubscribedRobots += numOfSubscribedRobots;

                    if (numOfSubscribedRobots > 0 and worksiteDiscoveryTimes[w] == 9999):
                        #-- first worksite discovery, remember it
                        worksiteDiscoveryTimes[w] = t;

                if (totalNumOfSubscribedRobots > 0):
                    if (totalNumOfSubscribedRobots >= numOfSites[t]):
                        idealRobotPercentagePerWorksite = 1 / numOfSites[t];
                    else:
                        idealRobotPercentagePerWorksite = 1/totalNumOfSubscribedRobots # not enough robots to cover all sites, there should be 1 robot per site

                    #-- second pass through worksites
                    for w in range(len(sitesSubscribedRobots[t])):
                        numOfSubscribedRobots = len(sitesSubscribedRobots[t][w]);
                        if (numOfSubscribedRobots > 0):

                            percentageOfSubscribedRobots = numOfSubscribedRobots / totalNumOfSubscribedRobots;
                            differencePercentageOfSubscribedRobotsFromIdeal = abs(idealRobotPercentagePerWorksite - percentageOfSubscribedRobots);
                            differencePercentageOfSubscribedRobotsFromIdealInRun.append(differencePercentageOfSubscribedRobotsFromIdeal)

                            #print(" t={} total subscribed robots {} subscribed robots {} ideal % {}   diff for site {} : {}".format(t,totalNumOfSubscribedRobots, sitesSubscribedRobots[t], idealRobotPercentagePerWorksite, w, differencePercentageOfSubscribedRobotsFromIdeal));

                #-- unemployment rate for this point in time
                unemploymentRatesInRun.append(1 - totalNumOfSubscribedRobots/numOfWorkingRobots);

                #--
                #print(" t={} subscribed robots {} unemployment rate {} broken robots {}".format(t,totalNumOfSubscribedRobots, unemploymentRatesInRun[t], brokenRobotIds[t]));


                #-- calculate foraging trip times
                if (t>0):

                    #print(sitesReachedRobots[t])

                    #-- find end of foraging trips from base to worksite (robots just arrived to worksite, after being on foraging trip for at least 1s )
                    for w in range(len(sitesReachedRobots[t])):
                        if (len(sitesReachedRobots[t][w]) > 0):
                            for robotId in sitesReachedRobots[t][w]:
                                if (not robotId in sitesReachedRobots[t-1][w]):
                                    robotIdIndex = robotIds.index(robotId);
                                    foragingTripTime = t-robotsForagingTripStartTimes[robotIdIndex]
                                    if (foragingTripTime > 0 and robotsForagingTripStartTimes[robotIdIndex] > 0):
                                        robotsForagingTripTimes.append(foragingTripTime);
                                        if (robotIdIndexForForagingTripTimeDebug < 0 or robotIdIndexForForagingTripTimeDebug == robotIdIndex):
                                            print(" t={} robot {} (index {}) ended foraging trip at worksite. Length: {} Reached now: {} Reached before: {}".format(t,robotId, robotIdIndex, foragingTripTime, sitesReachedRobots[t], sitesReachedRobots[t-1]))

                    #-- find new starts of foraging trips from worksite to base (robots just got resource)
                    for w in range(len(sitesLadenRobots[t])):
                        if (len(sitesLadenRobots[t][w]) > 0): #
                            for robotId in sitesLadenRobots[t][w]:
                                if (not robotId in sitesLadenRobots[t-1][w]):
                                    robotIdIndex = robotIds.index(robotId);
                                    robotsForagingTripStartTimes[robotIdIndex] = t;
                                    if (robotIdIndexForForagingTripTimeDebug < 0 or robotIdIndexForForagingTripTimeDebug == robotIdIndex):
                                        print(" t={} robot {} (index {}) started foraging trip from worksite. Laden now: {} Laden before: {}".format(t,robotId, robotIdIndex, sitesLadenRobots[t], sitesLadenRobots[t-1]))

                    #-- find end of foraging trips from worksite to base (robots just arrived to base)
                    for w in range(len(sitesEarningRobots[t])):
                        if (len(sitesEarningRobots[t][w]) > 0):
                            for robotId in sitesEarningRobots[t][w]:
                                if (not robotId in sitesEarningRobots[t-1][w]):
                                    #-- robot reached the base and ended one foraging trip
                                    robotIdIndex = robotIds.index(robotId);
                                    foragingTripTime = t-robotsForagingTripStartTimes[robotIdIndex]
                                    robotsForagingTripTimes.append(foragingTripTime)
                                    if (robotIdIndexForForagingTripTimeDebug < 0 or robotIdIndexForForagingTripTimeDebug == robotIdIndex):
                                        print(" t={} robot {} (index {}) ended foraging trip in base. Length: {} Earning now: {} Earning before: {}".format(t,robotId, robotIdIndex, foragingTripTime, sitesEarningRobots[t], sitesEarningRobots[t-1]))

                    #-- find start of foraging trips from base to worksite (robots just left base)
                    for w in range(len(sitesEarningRobots[t-1])):
                        if (len(sitesEarningRobots[t-1][w]) > 0):
                            for robotId in sitesEarningRobots[t-1][w]:
                                if (not robotId in sitesEarningRobots[t][w]):
                                    robotIdIndex = robotIds.index(robotId);
                                    robotsForagingTripStartTimes[robotIdIndex] = t;
                                    if (robotIdIndexForForagingTripTimeDebug < 0 or robotIdIndexForForagingTripTimeDebug == robotIdIndex):
                                        print(" t={} robot {} (index {}) started foraging trip from base. Earning now: {} Earning before: {}".format(t,robotId, robotIdIndex, sitesEarningRobots[t], sitesEarningRobots[t-1]))

                    #-- find when robots become unsubscribed and their foraging trip is interrupted
                    for w in range(len(sitesSubscribedRobots[t-1])):
                        if (len(sitesSubscribedRobots[t-1][w]) > 0):
                            for robotId in sitesSubscribedRobots[t-1][w]:
                                if (not robotId in sitesSubscribedRobots[t][w]):
                                    robotIdIndex = robotIds.index(robotId);
                                    robotsForagingTripStartTimes[robotIdIndex] = -1;
                                    if (robotIdIndexForForagingTripTimeDebug < 0 or robotIdIndexForForagingTripTimeDebug == robotIdIndex):
                                            print(" t={} robot {} (index {}) stopped foraging. Subscribed now: {} subscribed before: {}".format(t,robotId, robotIdIndex, sitesSubscribedRobots[t], sitesSubscribedRobots[t-1]))


        #-- calculate final stats for this run
        results_unemploymentRate.append(pyCreeper.crData.getAverageOfAList(unemploymentRatesInRun))
        results_unevenEmploymentRate.append(pyCreeper.crData.getAverageOfAList(differencePercentageOfSubscribedRobotsFromIdealInRun));
        results_foragingTripTravelTime.append(pyCreeper.crData.getAverageOfAList(robotsForagingTripTimes));
        results_averageWorksiteDiscoveryTime.append(pyCreeper.crData.getAverageOfAList(worksiteDiscoveryTimes));

        #-- sanity checks
        if (len(worksiteDiscoveryTimes) != numOfSites[0]):
            print("!!!! not all worksite were discovered in run {}".format(runNo))

    returnData = [results_unemploymentRate, results_unevenEmploymentRate, results_foragingTripTravelTime, results_averageWorksiteDiscoveryTime];

    #-- cache data
    cacheHelpers.cacheICRSecondaryTotals(returnData, dataPath_, numOfRuns_);

    return returnData;




#================================== HELPERS ================

def getRunICRDataOverTime(runDataPathPrefix_, runNo_, numOfRobots_, maxTime_, rewardGainRate_):
    """
    Wraps the icrfAnalyser.getRunICRDataOverTime by making sure the data has been cleaned
    """

    if (constants.USING_FOOTBOTS_SIMULATION_RESULTS):
        isForaging = True;
        if ("PERFORM" in runDataPathPrefix_):
            isForaging = False;
        filePath = (dataFormatHelpers.getBaseDataPath()+"/{}/Run{}_infoEvents").format(runDataPathPrefix_, runNo_);
    else:
        filePath = (dataFormatHelpers.getBaseDataPath()+"/{}/run{}_infoEvents").format(runDataPathPrefix_, runNo_);


    #-- try if cleaned data exists, if not create it
    try:
        infoEventsData = crData.fileToArray(filePath + "_cleaned.txt","\t", False);
        if (RECALC_CLEANED_DATA):
            raise IOError;
    except IOError:

        #-- data needs to be cleaned and saved first
        infoEventsData = crData.fileToArray(filePath + ".txt","\t", True);

        if (constants.USING_FOOTBOTS_SIMULATION_RESULTS):
            infoEventsData = footbotAnalysisHelpers.cleanInfoEventsData(infoEventsData, isForaging, debug_=True);
        else:
            infoEventsData = cleanInfoEventsData(infoEventsData, debug_=False);

        helpers.saveArrayToFile(infoEventsData, filePath + "_cleaned.txt")
        print("    Saved cleaned data to " + filePath + "_cleaned.txt")
        #--

    if (constants.USING_FOOTBOTS_SIMULATION_RESULTS):


        return icrfAnalyser.getRunICRDataOverTime(filePath + "_cleaned.txt", numOfRobots_, maxTime_, rewardGainRate_, not isForaging, 99);
    else:


        return icrfAnalyser.getRunICRDataOverTime(filePath + "_cleaned.txt", numOfRobots_, maxTime_, rewardGainRate_, False, 48);



def cleanInfoEventsDataForRuns(dataPathPrefix_, numOfRuns_):
    """
    Clean info events data of specific runs and save it into separate files
    :param dataPathPrefix_:
    :param numOfRuns_:
    :return:
    """
    for runNo in range(numOfRuns_):
        runDataPathPrefix = "{}/run{}".format(dataPathPrefix_, runNo);
        filePath = (dataFormatHelpers.getBaseDataPath()+"/{}_infoEvents").format(runDataPathPrefix);
        print(">>> Cleaning info events data for {}...".format(filePath));

        #-- data needs to be cleaned and saved first
        infoEventsData = crData.fileToArray(filePath + ".txt","\t", True);
        infoEventsData = cleanInfoEventsData(infoEventsData);
        helpers.saveArrayToFile(infoEventsData, filePath + "_cleaned.txt")

def cleanInfoEventsData(infoEventsData_, falseReachedEventMaxTimeSpan_=5, debug_=False):
    """
    Get rid of rows of data that do not make sense, e.g. repeated scouting / abandonment etc.
    :param infoEventsData_:
    :return: cleaned array. in the same format as infoEventsData_
    """

    if (debug_):
        print(">>> cleaning info events data");

    returnData = [];
    shouldRemoveNextRow = False;
    lastRewardStartedEventRow = -1;
    lastSiteReachedEventRow = -1;
    lastSiteAbandonedEventRow = -1;
    for row in range(len(infoEventsData_)):
        isRowValid = True;

        time, eventType, siteId, robotId = icrfHelper.getInfoEventDataFromRow(infoEventsData_[row]);
        nextTime = -1;
        nextEventType = -1;
        nextSiteId = -1;
        nextRobotId = -1;
        prevTime = -1;
        prevEventType = -1;
        prevSiteId = -1;
        prevRobotId = -1;
        if (row < (len(infoEventsData_)-1)):
            nextTime, nextEventType, nextSiteId, nextRobotId = icrfHelper.getInfoEventDataFromRow(infoEventsData_[row+1]);
        if (row > 0):
            prevTime, prevEventType, prevSiteId, prevRobotId = icrfHelper.getInfoEventDataFromRow(infoEventsData_[row-1]);

        #-- first check if it is already known that the row should be removed
        if (shouldRemoveNextRow):
            isRowValid = False;
            shouldRemoveNextRow = False;

        #-- keep track of last encountered events
        if (eventType == dataFormatHelpers.getEventDescriptionForType(constants.E_REWARD_STARTED)):
            lastRewardStartedEventRow = row;
        elif (eventType == dataFormatHelpers.getEventDescriptionForType(constants.E_SITE_REACHED)):
            lastSiteReachedEventRow = row;
        elif (eventType == dataFormatHelpers.getEventDescriptionForType(constants.E_SITE_ABANDONED)):
            lastSiteAbandonedEventRow = row;


        #-- check dummy event at the beginning of each e-puck experiment
        if (eventType == dataFormatHelpers.getEventDescriptionForType(constants.E_SITE_SWITCHED) and time == 0):
            isRowValid = False;

        #-- check for repeated reached - abandoned - scouted events
        if (nextTime - time <= falseReachedEventMaxTimeSpan_ and nextSiteId == siteId and nextRobotId == robotId):
            if (eventType == dataFormatHelpers.getEventDescriptionForType(constants.E_SITE_REACHED) and nextEventType == dataFormatHelpers.getEventDescriptionForType(constants.E_SITE_ABANDONED)):
                #-- look further into the future, because reached + abandoned might be a valid event in case the site is truly depleted
                if (row < (len(infoEventsData_)-2)):
                    nextNextTime, nextNextEventType, nextNextSiteId, nextNextRobotId = icrfHelper.getInfoEventDataFromRow(infoEventsData_[row+2]);
                    if (nextNextTime - nextTime <= falseReachedEventMaxTimeSpan_ and nextNextSiteId == siteId and nextNextRobotId == robotId and
                        (nextNextEventType == dataFormatHelpers.getEventDescriptionForType(constants.E_SITE_SCOUTED) or nextNextEventType == dataFormatHelpers.getEventDescriptionForType(constants.E_ROBOT_RECRUITED))):
                        isRowValid = False;

            elif (eventType == dataFormatHelpers.getEventDescriptionForType(constants.E_SITE_ABANDONED) and nextEventType == dataFormatHelpers.getEventDescriptionForType(constants.E_SITE_SCOUTED)):
                isRowValid = False;
                shouldRemoveNextRow = True;

        #-- check that reward received happens no earlier than the next reward started event
        if (eventType == dataFormatHelpers.getEventDescriptionForType(constants.E_REWARD_RECEIVED)):
            if (nextTime > time):
                infoEventsData_[row][icrfAnalyser.EVENTS_RECORD_TIME_COLUMN] = nextTime;
                if (debug_):
                    print("   adjusting reward received time from {} to {} for row {}".format(time,nextTime,infoEventsData_[row]))

        #-- check that reward started and reward finished events are at least 1 time step apart
        if (eventType == dataFormatHelpers.getEventDescriptionForType(constants.E_REWARD_FINISHED)):
            prevRewardStartedTime, prevRewardStartedEventType, prevRewardStartedSiteId, prevRewardStartedRobotId = icrfHelper.getInfoEventDataFromRow(infoEventsData_[lastRewardStartedEventRow]);
            if (siteId == prevRewardStartedSiteId and time == prevRewardStartedTime):
                infoEventsData_[row][icrfAnalyser.EVENTS_RECORD_TIME_COLUMN] += 1;
                if (debug_):
                    print("   adjusting reward finished time from {} to {} for row {}".format(infoEventsData_[lastRewardStartedEventRow][icrfAnalyser.EVENTS_RECORD_TIME_COLUMN],infoEventsData_[row][icrfAnalyser.EVENTS_RECORD_TIME_COLUMN],infoEventsData_[row]))

            #-- now check that the next events are not earlier than the new adjusted time
            if (nextTime > 0):
                continueEventSearch = True;
                counter = 1;
                while(continueEventSearch):
                    nextTime2, nextEventType2, nextSiteId2, nextRobotId2 = icrfHelper.getInfoEventDataFromRow(infoEventsData_[row+counter]);
                    if (nextTime2 == time):
                        infoEventsData_[row+counter][icrfAnalyser.EVENTS_RECORD_TIME_COLUMN] += 1;
                        if (debug_):
                            print("      also adjusting next row time: {}".format(infoEventsData_[row+counter]))
                        counter += 1
                    else:
                        continueEventSearch = False;


        #-- check that site started/reached and site paused events are at least 1 time step apart
        if (eventType == dataFormatHelpers.getEventDescriptionForType(constants.E_SITE_PAUSED)):
            if (siteId == prevSiteId and time == prevTime and (prevEventType == dataFormatHelpers.getEventDescriptionForType(constants.E_SITE_STARTED) or prevEventType == dataFormatHelpers.getEventDescriptionForType(constants.E_SITE_REACHED))):
                infoEventsData_[row][icrfAnalyser.EVENTS_RECORD_TIME_COLUMN] += 1;
                if (debug_):
                    print("   adjusting site paused time from {} to {} for row {}".format(infoEventsData_[row-1][icrfAnalyser.EVENTS_RECORD_TIME_COLUMN],infoEventsData_[row][icrfAnalyser.EVENTS_RECORD_TIME_COLUMN],infoEventsData_[row]))

        #-- check that a site abandoned event that happens after cart is loaded (i.e., before a reward received event has a site started event before it.
        if (eventType == dataFormatHelpers.getEventDescriptionForType(constants.E_SITE_ABANDONED)):
            if (nextEventType == dataFormatHelpers.getEventDescriptionForType(constants.E_REWARD_RECEIVED) and prevEventType != dataFormatHelpers.getEventDescriptionForType(constants.E_SITE_STARTED)):
                newRow = infoEventsData_[row].copy();
                newRow[icrfAnalyser.EVENTS_RECORD_EVENT_TYPE_COLUMN] = dataFormatHelpers.getEventDescriptionForType(constants.E_SITE_STARTED);
                returnData.append(newRow)
                if (debug_):
                    print("   appending site started event before abandonment by a loaded robot {}".format(newRow));

        #-- check that a site abandonment event that happens when cart is not laded (i.e. no resource left in the worksite) does not have a site started event before it
        if (eventType == dataFormatHelpers.getEventDescriptionForType(constants.E_SITE_STARTED)):
            if (nextEventType == dataFormatHelpers.getEventDescriptionForType(constants.E_SITE_ABANDONED)):
                if (row < (len(infoEventsData_)-2)):
                    nextNextTime, nextNextEventType, nextNextSiteId, nextNextRobotId = icrfHelper.getInfoEventDataFromRow(infoEventsData_[row+2]);
                    if (nextNextEventType != dataFormatHelpers.getEventDescriptionForType(constants.E_REWARD_RECEIVED)):
                        isRowValid = False;


        #-- check that a recruitment / scouting event is followed by either worksite reached or abandoned. If not, this event should not count as it represents a dummy event
        if (eventType == dataFormatHelpers.getEventDescriptionForType(constants.E_ROBOT_RECRUITED) or eventType == dataFormatHelpers.getEventDescriptionForType(constants.E_SITE_SCOUTED)):
            if not(nextEventType == dataFormatHelpers.getEventDescriptionForType(constants.E_SITE_REACHED) or nextEventType == dataFormatHelpers.getEventDescriptionForType(constants.E_SITE_ABANDONED)):
                isRowValid = False;
                if (debug_):
                    print("   removing recruitment / scouting row because not followed by site reached / abandoned ")

        #-- check a recruitment / scouting event is not preceded by a foraging cycle with no abandonment event
        if (eventType == dataFormatHelpers.getEventDescriptionForType(constants.E_ROBOT_RECRUITED) or eventType == dataFormatHelpers.getEventDescriptionForType(constants.E_SITE_SCOUTED)):
            if (lastSiteReachedEventRow > 0 and lastSiteAbandonedEventRow > 0):
                prevSiteReachedTime, prevSiteReachedEventType, prevSiteReachedSiteId, prevSiteReachedRobotId = icrfHelper.getInfoEventDataFromRow(infoEventsData_[lastSiteReachedEventRow]);
                prevSiteAbandonedTime, prevSiteAbandonedEventType, prevSiteAbandonedSiteId, prevSiteAbandonedRobotId = icrfHelper.getInfoEventDataFromRow(infoEventsData_[lastSiteAbandonedEventRow]);
                #print(prevSiteAbandonedTime, prevSiteReachedTime)
                if (prevSiteAbandonedTime < prevSiteReachedTime and prevSiteReachedRobotId == robotId):
                    newRow = infoEventsData_[lastSiteReachedEventRow].copy(); # most data copied from the last site reached event
                    newRow[icrfAnalyser.EVENTS_RECORD_TIME_COLUMN] = prevTime; # time of reward finished
                    newRow[icrfAnalyser.EVENTS_RECORD_EVENT_TYPE_COLUMN] = dataFormatHelpers.getEventDescriptionForType(constants.E_SITE_ABANDONED);
                    returnData.append(newRow)
                    if (debug_):
                        print("   adding abandonment event when new site was scouted / recruited to: {}".format(newRow))

        #-- include this row if valid
        if (isRowValid):
            returnData.append(infoEventsData_[row]);
        else:
            if (debug_):
                print ("   removing row {}: {} ".format(row, infoEventsData_[row]))

    return returnData;




#======================= DATA CONSISTENCY
def getDataConsistencyInfo(dataPath_, numOfRuns_, numOfRobots_):
    """

    :param dataPath_:
    :param numOfRuns_:
    :param numOfRobots_:
    :return: [results_percentageOfRunsWithRobotFailures (in percentage, out of total number of runs,
            results_numOfRobotFailures,
            results_timeOfRobotFailures (in percentage of corresponding run time),
            results_recruitmentVsScouting (percentage of recruitment events out of all info gain events)]
    """

    results_numOfRobotFailures = [];
    results_timeOfRobotFailures = [];
    results_recruitmentVsScouting = [];
    results_percentageOfRunsWithRobotFailures = 0;


    #-- calculate data
    completionTimes = getCompletionTimes(dataPath_, numOfRuns_, convertToMinutes_=False);

    for runNo in range(numOfRuns_):
        runData = crData.fileToArray((dataFormatHelpers.getBaseDataPath()+"/{}/run{}_infoEvents.txt").format(dataPath_, runNo),"\t", True);

        numRecruitmentEvents = 0;
        numInfoGainEvents = 0;
        numOfRobotFailures = 0;
        timeOfRobotFailures = 0;
        recruitmentVsScouting = 0;

        for i in range(len(runData)):
            eventType = runData[i][1];
            time = runData[i][0];

            if (eventType == constants.E_ROBOT_BROKE):
                numOfRobotFailures += 1;

                #-- always, the robot fixed will be the following line
                fixedTime = runData[i+1][0];
                #print("Run {}. failure started at t={} finished at t={}".format(runNo, time, fixedTime));
                if (fixedTime > completionTimes[runNo]):
                    fixedTime = completionTimes[runNo];
                    #print ("   ! adjusted to run completion time={}".format(completionTimes[runNo]));
                failureTime = fixedTime - time;
                failureTimeAdj = failureTime / (1.0*completionTimes[runNo]*numOfRobots_) * 100;
                #print("   > failure time={} adjusted for completion time {} = {}".format(failureTime,completionTimes[runNo],failureTimeAdj));
                timeOfRobotFailures += failureTimeAdj; # as percentage from total completion time, per robot

            if (eventType == constants.E_ROBOT_RECRUITED):
                numInfoGainEvents += 1;
                numRecruitmentEvents += 1;
            elif (eventType == constants.E_SITE_SCOUTED):
                numInfoGainEvents += 1;

        #print("Run {} info gain events {}  recruitment events {}".format(runNo, numInfoGainEvents, numRecruitmentEvents))
        recruitmentVsScouting = numRecruitmentEvents / (numInfoGainEvents * 1.0) * 100;

        results_numOfRobotFailures.append(numOfRobotFailures);
        results_timeOfRobotFailures.append(timeOfRobotFailures);
        results_recruitmentVsScouting.append(recruitmentVsScouting);
        if (numOfRobotFailures > 0):
            results_percentageOfRunsWithRobotFailures += (1.0/numOfRuns_)*100;

    return ([results_percentageOfRunsWithRobotFailures, results_numOfRobotFailures, results_timeOfRobotFailures, results_recruitmentVsScouting])