collectFixedChannels.py

'''
This code gets the p2m files generated by InSite via simulation.py and writes the ray information in files.
Different from todb.py, it does not use a database (no episode.db).
One does not need to specify the number of scenes per episode because this information is obtained from
the JSON file and confirmed (redundancy) with the file 'sumoOutputInfoFileName.txt' at "run_dir".

If InSite version 3.2 or earlier was used, the phase needs to be obtained from .cir files. In this
case, this script uses class P2mCir in file p2mcir.py to obtain the phase. If version 3.3 or later is used,
then the phase is part of the .p2m files.
'''
import os
import json
import numpy as np
import csv
from sys import argv
import h5py

#From https://coderwall.com/p/x6xtxq/convert-bytes-to-int-or-int-to-bytes-in-python
def bytes_to_int(bytes):
    result = 0
    for b in bytes:
        result = result * 256 + int(b)
    return result

def int_to_bytes(value, length):
    result = []
    for i in range(0, length):
        result.append(value >> (i * 8) & 0xff)
    result.reverse()
    return result

# now we don't need to recall config.py. We can simply specify folders below
# import config as c

from rwisimulation.tfrecord import SceneNotInEpisodeSequenceError
#from rwisimulation.tfrecord import UnexpectedCarsWithAntennaChangeError, EpisodeNotStartingFromZeroError
# from rwimodeling import objects
from rwiparsing import P2mPaths
from rwiparsing import P2mCir

if len(argv) != 3:
    print('You need to specify the input folder (that has the output files written by the ray-tracing simulator) and output folder!')
    print('Usage: python', argv[0], 'input_folder output_folder')
    exit(-1)

numScenesPerEpisode = 200 #50, 1000, etc
numTxRxPairsPerScene = 10
numRaysPerTxRxPair = 100
numParametersPerRay = 7 + 1 #has the ray angle now
max_num_interactions = 9 #this will be checked in this script, so you can start by guessing and then re-run the script
should_write_interactions = False
should_write_npz_file = False

#if needed, manually create the output folder
output_folder = argv[2]

os.makedirs(output_folder, exist_ok=True)

fileNamePrefix = os.path.join(output_folder, 'urban_canyon_v2i_5gmv1_rays') #prefix of output files
if should_write_npz_file:
    pythonExtension = '.npz'
matlabExtension = '.hdf5'

# from rwisimulation.datamodel import save5gmdata as fgdb

if not os.path.exists(output_folder):
    os.mkdir(output_folder)

def base_run_dir_fn(i):  # the folders will be run00001, run00002, etc.
    """returns the `run_dir` for run `i`"""
    return "run{:05d}".format(i)


last_simulation_info = None
simulation_info = None
# session = fgdb.Session()

# Object which will be modified in the RWI project
# base_insite_project_path = 'D:/insitedata/insite_new_simuls/'
# Folder to store each InSite project and its results (will create subfolders for each "run", run0000, run0001, etc.)
results_dir = argv[1]  # 'D:/owncloud-lasse/5GM_DATA/flat_simulation/results_new_lidar/'

# The info below typically does not change
# dst_object_file_nameBaseName = "random-line.object"
# Ray-tracing output folder (where InSite will store the results (Study Area name)).
# They will be later copied to the corresponding output folder specified by results_dir
project_output_dirBaseName = 'study'
# Name (basename) of the paths file generated in the simulation
paths_file_name = 'model.paths.t001_01.r002.p2m'
# Output files, which are written by the Python scripts
# Name (basename) of the JSON output simulation info file
simulation_info_file_name = 'wri-simulation.info'

this_scene_i = 0  #indicates scene number within episode
total_num_scenes = 0 #all processed scenes
ep_i = -1  # it's summed to 1 and we need to start by 0
actual_max_num_interactions = -np.Infinity #get the max number of interactions
#episode = None
# n_run = 100000
should_stop = False
while not should_stop:
    at_least_one_valid_scene_in_this_episode = False
    #gains, phases, etc. for gains
    allEpisodeData = np.zeros((numScenesPerEpisode, numTxRxPairsPerScene, numRaysPerTxRxPair,
                               numParametersPerRay), np.float32)
    allEpisodeData.fill(np.nan)
    if should_write_interactions:
        #positions (x,y,z) of interactions (see Table 20.1: Propagation Path Interactions) of InSite Reference Manual
        allInteractionsPositions = np.zeros((numScenesPerEpisode, numTxRxPairsPerScene, numRaysPerTxRxPair,
                                   max_num_interactions, 3), np.float32) #3 because (x,y,z)
        allInteractionsPositions.fill(np.nan)
        # Strings
        allInteractionsDescriptions = np.zeros((numScenesPerEpisode, numTxRxPairsPerScene, numRaysPerTxRxPair,
                                                max_num_interactions, 2), dtype=int) #2 because there are at most 2 letters
        #number of interactions per ray
        allInteractionsNumbers = np.zeros((numScenesPerEpisode, numTxRxPairsPerScene, numRaysPerTxRxPair), dtype=np.int64)
        allInteractionsNumbers.fill(-1)

    for s in range(numScenesPerEpisode):
        # for run_i in range(100): # use the number of examples in config.py
        run_dir = os.path.join(results_dir, base_run_dir_fn(total_num_scenes))
        # object_file_name = os.path.join(run_dir, dst_object_file_nameBaseName)
        # rays information but phase
        abs_paths_file_name = os.path.join(run_dir, project_output_dirBaseName, paths_file_name)
        if os.path.exists(abs_paths_file_name) == False:
            print('\nWarning: could not find file ', abs_paths_file_name, ' Stopping...')
            should_stop = True
            break
        # now we get the phase info from CIR file
        abs_cir_file_name = abs_paths_file_name.replace("paths", "cir")  # name for the impulse response (cir) file
        if os.path.exists(abs_cir_file_name) == False:
            print('ERROR: could not find file ', abs_cir_file_name)
            print('Did you ask InSite to generate the impulse response (cir) file?')
            exit(-1)

        abs_simulation_info_file_name = os.path.join(run_dir, simulation_info_file_name)
        with open(abs_simulation_info_file_name) as infile:
            simulation_info = json.load(infile)

        # start of episode
        if simulation_info['scene_i'] == 0:
            ep_i += 1
            this_scene_i = 0  # reset counter
            # if episode is not None:
            #    session.add(episode)
            #    session.commit()

            # read SUMO information for this scene from text CSV file
            sumoOutputInfoFileName = os.path.join(run_dir, 'sumoOutputInfoFileName.txt')
            with open(sumoOutputInfoFileName, 'r') as f:
                #read head, which is surrounded by quotes:
                #"episode_i,scene_i,receiverIndex,veh,veh_i,typeID,xinsite,yinsite,x3,y3,z3,lane_id,angle,speed,length, width, height,distance,waitTime,currentTime(ms)=70050,Ts(s)=0.05"
                #Hence, the CSV does not split them because the row is protected by " " and is interpreted as 1 string
                #print(sumoAllLines[0][-1],'ss')
                sumoReader = csv.reader(f)
                for row in sumoReader:
                    headerItems = row[0].split(',')
                    TsString = headerItems[-1]
                    try:
                        Ts = TsString.split('=')[1]
                        timeString = headerItems[-2]
                        time = timeString.split('=')[1]
                    except IndexError:  # old format
                        Ts = 0.005  # initialize values
                        time = -1
                    break  # process only first 2 rows / line AK-TODO should eliminate the loop
                for row in sumoReader:
                    # secondRow = row[1].split(',')
                    thisEpisodeNumber = int(row[0])
                    if thisEpisodeNumber != ep_i:
                        print('ERROR: thisEpisodeNumber != ep_i. They are:', thisEpisodeNumber, 'and', ep_i,
                              'file: ', sumoOutputInfoFileName, 'read:', row)
                        exit(1)
                    break  # process only first 2 rows / line AK-TODO should eliminate the loop
            # episode = fgdb.Episode(
            #    insite_pah=run_dir,
            #    sumo_path=sumoOutputInfoFileName,
            #    simulation_time_begin=time, #in milliseconds
            #    sampling_time=Ts, #in seconds
            # )

        #if episode is None:
        #    raise EpisodeNotStartingFromZeroError("From file {}".format(object_file_name))

        if simulation_info['scene_i'] != this_scene_i:
            raise SceneNotInEpisodeSequenceError('Expecting {} found {}'.format(
                this_scene_i,
                simulation_info['scene_i'],
            ))

        # with open(object_file_name) as infile:
        #    obj_file = objects.ObjectFile.from_file(infile)
        print(abs_paths_file_name)  # AK TODO take out this comment and use logging
        paths = P2mPaths(abs_paths_file_name)
        cir = P2mCir(abs_cir_file_name)

        # scene = fgdb.Scene()
        # scene.study_area = ((0, 0, 0), (0, 0, 0))
        for txrx_pair_i in range(numTxRxPairsPerScene): #if num Tx = 1, then receiver_number is the receiver index
            if paths.get_total_received_power(txrx_pair_i + 1) is not None:
                total_received_power = paths.get_total_received_power(txrx_pair_i + 1)
                mean_time_of_arrival = paths.get_mean_time_of_arrival(txrx_pair_i + 1)
                #receiver.position = object.position

                sixParameters=paths.get_6_parameters_for_all_rays(txrx_pair_i + 1)
                numRays = sixParameters.shape[0]
                areLOSChannels = paths.is_los(txrx_pair_i + 1)
                phases = cir.get_phase_ndarray(txrx_pair_i + 1)  # get phases for all rays in degrees
                #go from 0:numRays to support a number of valid rays smaller than the maximum
                allEpisodeData[this_scene_i, txrx_pair_i, 0:numRays, 0:6] = sixParameters
                #allEpisodeData[this_scene_i][rec_i] = sixParameters
                allEpisodeData[this_scene_i, txrx_pair_i, 0:numRays, 6] = areLOSChannels
                if numParametersPerRay == 8:
                    allEpisodeData[this_scene_i, txrx_pair_i, 0:numRays, 7] = phases

                if should_write_interactions:
                    interactions_strings = paths.get_interactions_list(txrx_pair_i + 1)
                    for ray_i in range(numRays):
                        #interactions positions
                        interactions_positions=paths.get_interactions_positions(txrx_pair_i + 1, ray_i + 1)
                        theseInteractions = interactions_strings[ray_i].split('-')
                        num_interactions = len(interactions_positions)
                        allInteractionsNumbers[this_scene_i, txrx_pair_i, ray_i] = num_interactions #keep the number of interactions
                        if num_interactions > actual_max_num_interactions:
                            actual_max_num_interactions = num_interactions #update
                        if num_interactions > max_num_interactions:
                            print('ERROR: Found num of interactions = ', num_interactions, 'while you specified the maximum is', max_num_interactions)
                            exit(-1)
                        for interaction_i in range(num_interactions):
                            allInteractionsPositions[this_scene_i, txrx_pair_i, ray_i, interaction_i] = interactions_positions[interaction_i]
                            stringAsBytes = theseInteractions[interaction_i].encode() #https://www.mkyong.com/python/python-3-convert-string-to-bytes/
                            allInteractionsDescriptions[this_scene_i, txrx_pair_i, ray_i, interaction_i, 0] = int(stringAsBytes[0])
                            if len(stringAsBytes) > 1:
                                allInteractionsDescriptions[this_scene_i, txrx_pair_i, ray_i, interaction_i, 1] = int(stringAsBytes[1])

                        # episode.scenes.append(scene)
                at_least_one_valid_scene_in_this_episode = True #indicate this episode has at least one valid scene

        print('\rProcessed episode: {} scene: {}, total {} '.format(ep_i, this_scene_i, total_num_scenes), end='')
        this_scene_i += 1
        total_num_scenes += 1  # increment loop counter

    if at_least_one_valid_scene_in_this_episode:
        if should_write_npz_file:
            outputFileName = fileNamePrefix + '_e' + str(ep_i) + pythonExtension
            np.savez(outputFileName, allEpisodeData=allEpisodeData)
            print('==> Wrote file ' + outputFileName)

        if should_write_interactions:
            if should_write_npz_file:
                outputFileName = fileNamePrefix + '_e' + str(ep_i) + '_interactions' + pythonExtension
                np.savez(outputFileName, allInteractionsPositions=allInteractionsPositions, \
                         allInteractionsDescriptions = allInteractionsDescriptions, allInteractionsNumbers=allInteractionsNumbers)
                print('==> Wrote file ' + outputFileName)

        outputFileName = fileNamePrefix + '_e' + str(ep_i) + matlabExtension
        print('==> Wrote file ' + outputFileName)
        f = h5py.File(outputFileName, 'w')
        f['allEpisodeData'] = allEpisodeData
        f.close()

        if should_write_interactions:
            #because they are large arrays, store interactions in another file
            outputFileName = fileNamePrefix + '_e' + str(ep_i) + '_interactions' + matlabExtension
            print('==> Wrote file ' + outputFileName)
            f = h5py.File(outputFileName, 'w')
            f['allInteractionsPositions'] = allInteractionsPositions
            f['allInteractionsDescriptions'] = allInteractionsDescriptions
            f['allInteractionsNumbers'] = allInteractionsNumbers
            f.close()

print()
print('Processed ', total_num_scenes, ' scenes (RT simulations)')
if should_write_interactions and max_num_interactions != actual_max_num_interactions:
    print('Found a max num of interactions = ', actual_max_num_interactions, 'while you specified = ', max_num_interactions)
    print('Maybe you can consider re-running in case you do not want to waste some space in the array that stores interaction strings')