From 5ce360b7fc5e561faeb7c6d552fe677ae4d6c393 Mon Sep 17 00:00:00 2001 From: Catarina Alves Date: Fri, 11 Mar 2022 01:12:55 +0000 Subject: [PATCH 01/13] Updated demo notebook: example/Demo_make_simlibs.ipynb - demo notebook updated for the latest version - correct small typo - correct last typo - update version --- example/Demo_make_simlibs.ipynb | 55 ++++++++++++++------------------- 1 file changed, 24 insertions(+), 31 deletions(-) diff --git a/example/Demo_make_simlibs.ipynb b/example/Demo_make_simlibs.ipynb index 389cc3c..996350a 100644 --- a/example/Demo_make_simlibs.ipynb +++ b/example/Demo_make_simlibs.ipynb @@ -11,12 +11,12 @@ "The following commands can be run from the terminal or from a notebook.\n", "\n", "## Index\n", - "* [Options](#options)\n", - "* [Example of usage](#usage)\n", + "1. [Options](#options)\n", + "2. [Example of usage](#usage)\n", " 1. [Download the desired file](#download)\n", - " * [Set up options](#chooseOptions)\n", - " * [Run the script](#run)\n", - " * [Output](#output)\n", + " 2. [Set up options](#chooseOptions)\n", + " 3. [Run the script](#run)\n", + " 4. [Output](#output)\n", "\n", "## Options\n", "\n", @@ -42,22 +42,22 @@ "\n", "### 1. Download the desired file\n", "\n", - "In this example, we will download the `baseline_v1.4_10yrs.db` file from [here](https://lsst-web.ncsa.illinois.edu/sim-data/) via the use of the nix Command Line `curl` tool. After downloaded, I will move it to the Downloads folder in order to showcase some more options of `make_simlibs`.\n", + "In this example, we will download the `baseline_v2.0_10yrs.db` file from [here](https://epyc.astro.washington.edu/~lynnej/opsim_downloads/) via the use of the nix Command Line `curl` tool. After downloaded, I will move it to the Downloads folder in order to showcase some more options of `make_simlibs`.\n", "\n", "```shell\n", - "curl -O https://lsst.ncsa.illinois.edu/sim-data/sims_featureScheduler_runs1.4/baseline/baseline_v1.4_10yrs.db\n", + "curl -O https://epyc.astro.washington.edu/~lynnej/opsim_downloads/fbs_2.0/baseline/baseline_v2.0_10yrs.db\n", "\n", - "mv baseline_v1.4_10yrs.db ${HOME}/Downloads/\n", + "mv baseline_v2.0_10yrs.db ${HOME}/Downloads/\n", "\n", - "# The database file is now in ${HOME}/Downloads/baseline_v1.4_10yrs.db\n", + "# The database file is now in ${HOME}/Downloads/baseline_v2.0_10yrs.db\n", "```\n", "\n", "### 2. Set up options\n", "\n", "- `data_root` $\\rightarrow$ path to the directory containing `dbname`. In this case the package is in `${HOME}`.\n", - "- `dbname` $\\rightarrow$ relative path to the OpSim DataBase file from `data_root`. In this case it is `Downloads/baseline_v1.4_10yrs.db`. (This could have also been done as `data_root` $\\rightarrow$ `${HOME}/Downloads` and `dbname` $\\rightarrow$ `baseline_v1.4_10yrs.db` as used below.)\n", + "- `dbname` $\\rightarrow$ relative path to the OpSim DataBase file from `data_root`. In this case it is `Downloads/baseline_v2.0_10yrs.db`. (This could have also been done as `data_root` $\\rightarrow$ `${HOME}/Downloads` and `dbname` $\\rightarrow$ `baseline_v2.0_10yrs.db` as used below.)\n", "\n", - "- `opsimversion` $\\rightarrow$ version of opsim used. Here we used `fbsv1p3`.\n", + "- `opsimversion` $\\rightarrow$ version of opsim used. Here we used `fbsv2`.\n", "\n", "If no `ddf_simlibfilename` and `wfd_simlibfilename` paths are given, the generated files will be output in the current working directory given by `PWD`.\n", "\n", @@ -72,7 +72,7 @@ "We can then run the script\n", "\n", "```shell\n", - "python make_simlibs.py --data_root /Users/user_name/Downloads --dbname baseline_v1.4_10yrs.db --opsimversion 'fbsv1p3'\n", + "make_simlibs.py --dbname baseline_v2.0_10yrs.db --data_root /Users/user_name/Downloads --opsimversion fbsv2\n", "```\n", "\n", "All the above following commands can be run from the terminal but they can also be run from a notebook by adding `!` before the command. Ex:" @@ -93,16 +93,16 @@ "source": [ "#### Alternatives\n", "\n", - "The `make_simlibs` script prints several messages while running, and can take some time to run. Hence, it is often convenient to run in the background, and store the output in a log file. Naming the log file `simlib_v1.4.log`, the command to run the script then becames:\n", + "The `make_simlibs` script prints several messages while running, and can take some time to run. Hence, it is often convenient to run in the background, and store the output in a log file. Naming the log file `simlib_v2.0.log`, the command to run the script then becames:\n", "\n", "```shell\n", - "python make_simlibs.py --data_root /Users/user_name/Downloads --dbname baseline_v1.4_10yrs.db --opsimversion 'fbsv1p3' > simlib_v1.4.log 2>&1 &\n", + "python make_simlibs.py --dbname baseline_v2.0_10yrs.db --data_root /Users/user_name/Downloads --opsimversion fbsv2 > simlib_v2.0.log 2>&1 &\n", "```\n", "\n", "If you want to leave this running on a remote machine\n", "\n", "```shell\n", - "nice nohup python make_simlibs.py --data_root /Users/user_name/Downloads --dbname baseline_v1.4_10yrs.db --opsimversion 'fbsv1p3' > simlib_v1.4.log 2>&1 &\n", + "nice nohup python make_simlibs.py --dbname baseline_v2.0_10yrs.db --data_root /Users/user_name/Downloads --opsimversion fbsv2 > simlib_v2.0.log 2>&1 &\n", "```\n", "should do the job.\n", "\n", @@ -112,15 +112,15 @@ "Using the above command, the output files will be in the `~/OpSimSummary/scripts` folder.\n", "\n", "The files generated are:\n", - "- `baseline_v1.4_10yrs_DDF.simlib`\n", - "- `baseline_v1.4_10yrs_DDF_avail.csv`\n", - "- `baseline_v1.4_10yrs_DDF_sel.csv`\n", - "- `baseline_v1.4_10yrs_WFD.simlib`\n", - "- `baseline_v1.4_10yrs_WFD_avail.csv`\n", - "- `baseline_v1.4_10yrs_WFD_sel.csv`\n", - "- `ddf_minion_1016_sqlite.csv`\n", - "- `simlib_v1.4.log`\n", - "- `wfd_minion_1016_sqlite.csv`\n", + "- `baseline_v2.0_10yrs_DDF.simlib`\n", + "- `baseline_v2.0_10yrs_DDF_avail.csv`\n", + "- `baseline_v2.0_10yrs_DDF_sel.csv`\n", + "- `baseline_v2.0_10yrs_WFD.simlib`\n", + "- `baseline_v2.0_10yrs_WFD_avail.csv`\n", + "- `baseline_v2.0_10yrs_WFD_sel.csv`\n", + "- `testingDDF`\n", + "- `simlib_v2.0.log`\n", + "- `testingWFD`\n", "\n", "\n", "The files ending in `simlib` are the files that can be used as input to `SNANA`. If appropriate for the use, one can further using the `simlib_coadd` script of `SNANA`, which can coadd observations over time intervals to produce more compact simlib files that help `SNANA` generate faster simulations.\n", @@ -130,13 +130,6 @@ "\n", "Go to [Index](#index)." ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From 3986f60a978c37a9fa651b55ecb0529c08466205 Mon Sep 17 00:00:00 2001 From: "R.Biswas" Date: Fri, 9 Dec 2022 22:27:16 +0100 Subject: [PATCH 02/13] WIP : comments only modified: opsim_out.py --- opsimsummary/opsim_out.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/opsimsummary/opsim_out.py b/opsimsummary/opsim_out.py index 60e2fa0..aa5a321 100644 --- a/opsimsummary/opsim_out.py +++ b/opsimsummary/opsim_out.py @@ -31,13 +31,18 @@ class OpSimOutput(object): version of OpSim corresponding to the output format. summary: `pd.DataFrame` selected records from the Summary Table of pointings + #### Enable propIDDict None propIDDict: dict dictionary with strings as keys and integers used in the Summary Table to denote these proposals - proposalTable: `pd.DataFrame` + + #### Enable proposalTable None + proposalTable: `pd.DataFrame`, the propsal table in the output subset: string subset of proposals included in this class + + #### Enable propIDDict None propIDs : list of integers integers corresponding to the subset selected through proposals """ From 6ec40b864b455fb3b5466679e679c6944deb17e6 Mon Sep 17 00:00:00 2001 From: "R.Biswas" Date: Sat, 10 Dec 2022 11:30:31 +0100 Subject: [PATCH 03/13] WIP : Adding code, if propIDDict is `None` we will only have `WFD` The `OpsimVars` variable is now changed (see below) modified: opsim_out.py propName=None, propIDName=None, propIDNameInSummary=None, ops_wfdname='WFD', ops_ddfname=None, --- opsimsummary/opsim_out.py | 54 ++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/opsimsummary/opsim_out.py b/opsimsummary/opsim_out.py index aa5a321..b8a0030 100644 --- a/opsimsummary/opsim_out.py +++ b/opsimsummary/opsim_out.py @@ -104,15 +104,12 @@ def __init__(self, summary, propIDDict=None, proposalTable=None, print('summary table has nans, exiting\n') sys.exit(1) - ddfPropID = list(self.propIDDict['ddf']) - ddfidx = summary.query('propID == @ddfPropID').index + if propIDDict is not None: + ddfPropID = list(self.propIDDict['ddf']) + ddfidx = summary.query('propID == @ddfPropID').index - # opsimVars['pointingRA'] / 'pointingDec' are native variables in the OpSim tables - # denoting RA and Dec. These are usually in degrees (opsimVars['angleUnit']) - # However, we only work with ditheredRA and ditheredDec, hence these variables have to - # be assigned. - summary.loc[ddfidx, 'ditheredRA'] = summary.loc[ddfidx, self.opsimVars['pointingRA']] - summary.loc[ddfidx, 'ditheredDec'] = summary.loc[ddfidx, self.opsimVars['pointingDec']] + summary.loc[ddfidx, 'ditheredRA'] = summary.loc[ddfidx, self.opsimVars['pointingRA']] + summary.loc[ddfidx, 'ditheredDec'] = summary.loc[ddfidx, self.opsimVars['pointingDec']] # Have a clear unambiguous ra, dec in radians following LSST convention # These are the columns `_ra`, `_dec` which should have the dithered @@ -237,19 +234,34 @@ def get_opsimVariablesForVersion(opsimversion='fbs2'): # Feature Based Scheduler : version 2.x elif opsimversion == 'fbsv2': - x = dict(summaryTableName='observations', - obsHistID='observationId', - propName='proposalType', - propIDName='proposalId', - propIDNameInSummary='proposalId', - ops_wfdname='WFD', - ops_ddfname='DDF', - expMJD='observationStartMJD', - FWHMeff='seeingFwhmEff', - pointingRA='fieldRA', - pointingDec='fieldDec', - filtSkyBrightness='skyBrightness', - angleUnit='degrees') + if propIDDict is not None: + x = dict(summaryTableName='observations', + obsHistID='observationId', + propName='proposalType', + propIDName='proposalId', + propIDNameInSummary='proposalId', + ops_wfdname='WFD', + ops_ddfname='DDF', + expMJD='observationStartMJD', + FWHMeff='seeingFwhmEff', + pointingRA='fieldRA', + pointingDec='fieldDec', + filtSkyBrightness='skyBrightness', + angleUnit='degrees') + else: + x = dict(summaryTableName='observations', + obsHistID='observationId', + propName=None, + propIDName=None, + propIDNameInSummary=None, + ops_wfdname='WFD', + ops_ddfname=None, + expMJD='observationStartMJD', + FWHMeff='seeingFwhmEff', + pointingRA='fieldRA', + pointingDec='fieldDec', + filtSkyBrightness='skyBrightness', + angleUnit='degrees') else: raise NotImplementedError('`get_opsimVariablesForVersion` is not implemented for this `opsimversion`') From aedcbec729d63b26a2cbbd18ac5f6cd74f64a819 Mon Sep 17 00:00:00 2001 From: "R.Biswas" Date: Sat, 10 Dec 2022 14:51:30 +0100 Subject: [PATCH 04/13] WIP : Made changes to opsim_out.py to allow the use of no_proposal modified: opsim_out.py --- opsimsummary/opsim_out.py | 73 ++++++++++++++++++++++++++++++--------- 1 file changed, 56 insertions(+), 17 deletions(-) diff --git a/opsimsummary/opsim_out.py b/opsimsummary/opsim_out.py index b8a0030..b7ef2c4 100644 --- a/opsimsummary/opsim_out.py +++ b/opsimsummary/opsim_out.py @@ -45,10 +45,15 @@ class OpSimOutput(object): #### Enable propIDDict None propIDs : list of integers integers corresponding to the subset selected through proposals + + use_proposal_table : Bool, defaults to True + Do not use a proposal table, and works if no ProposalTable exists. + This is only implemented for opsimversion = `fsbv2` """ def __init__(self, summary, propIDDict=None, proposalTable=None, subset=None, propIDs=None, - opsimversion='fbsv2'): + opsimversion='fbsv2', + use_proposal_table=True): """ Constructor for the `OpSimOutput` class @@ -89,9 +94,13 @@ def __init__(self, summary, propIDDict=None, proposalTable=None, and `lsstv4` refers to outputs available from OpSim version 4. `fbs` refers to Feature Based Scheduler, with versions 1.x differentiated from versions 2.x + use_proposal_table : Bool, defaults to True + Do not use a proposal table, and works if no ProposalTable exists. + This is only implemented for opsimversion = `fsbv2` """ self.opsimversion = opsimversion self._opsimvars = None + self.use_proposal_table = use_proposal_table self.allowed_subsets = self.get_allowed_subsets(opsimversion) self.subset = subset self.propIDDict = propIDDict @@ -142,7 +151,8 @@ def __init__(self, summary, propIDDict=None, proposalTable=None, self._propID = propIDs @staticmethod - def get_opsimVariablesForVersion(opsimversion='fbs2'): + def get_opsimVariablesForVersion(opsimversion='fbs2', + use_proposal_table=True): """Static method to returns a dictionary for the opsim version where the keys are names of quantities used in this codebase, and the values are the names of quantities in the OpSim output database @@ -152,6 +162,11 @@ def get_opsimVariablesForVersion(opsimversion='fbs2'): opsimversion: string, defaults to `fbsv2` can be {`fbsv1`|`fbsv2`} + + use_proposal_table : Bool, defaults to True + whether we should use a proposal table. Only works with opsimversion + fsb2 + Returns ------- dictionary: key, value pairs where keys are variable names used in `OpSimSummary` @@ -170,6 +185,9 @@ def get_opsimVariablesForVersion(opsimversion='fbs2'): 'pointingRA': 'ditheredRA', 'pointingDec': 'ditheredDec', 'filtSkyBrightness': 'skyBrightness', 'angleUnit': 'degrees'} """ + if opsimversion not in ('fbsv2'): + if use_proposal_table == False: + raise NotImplementedError(f'Not having a proposal table is not implemented for opsimversion = {opsimversion}') if opsimversion == 'lsstv3': x = dict(summaryTableName='Summary', @@ -234,7 +252,7 @@ def get_opsimVariablesForVersion(opsimversion='fbs2'): # Feature Based Scheduler : version 2.x elif opsimversion == 'fbsv2': - if propIDDict is not None: + if use_proposal_table: x = dict(summaryTableName='observations', obsHistID='observationId', propName='proposalType', @@ -414,6 +432,7 @@ def fromOpSimDB(cls, dbname, user_propIDs=None, # dithercolumns=None, # add_dithers=False, + use_proposal_table=False, tableNames=('Summary', 'Proposal'), filterNull=False, **kwargs): @@ -452,15 +471,20 @@ def fromOpSimDB(cls, dbname, # rather than the property, but note that the property is calculated # through this method. So this gives the same thing dithercolumns = None - opsimVars = cls.get_opsimVariablesForVersion(opsimversion) + opsimVars = cls.get_opsimVariablesForVersion(opsimversion, + use_proposal_table) # Set tablenames ### Note to self : Proposal has not changed yet, and is thus ### not part of opsimVars - tableNames = (opsimVars['summaryTableName'], 'Proposal') + if use_proposal_table: + tableNames = (opsimVars['summaryTableName'], 'Proposal') + else: + tableNames = (opsimVars['summaryTableName']) # Check that subset parameter is legal - allowed_subsets = cls.get_allowed_subsets(opsimversion) + allowed_subsets = cls.get_allowed_subsets(opsimversion, + use_proposal_table) subset = subset.lower() if subset not in allowed_subsets: raise NotImplementedError('subset {} not implemented'.\ @@ -474,10 +498,14 @@ def fromOpSimDB(cls, dbname, propDict, propIDs, proposals = cls._get_propIDs(tableNames, engine, opsimversion, subset, - user_propIDs=user_propIDs) + user_propIDs=user_propIDs, + use_proposal_table=use_proposal_table) # Read the observations - summary = cls._read_summary_table_raw(engine, opsimVars, propIDs, subset) + summary = cls._read_summary_table_raw(engine, + opsimVars, + propIDs, + subset) if len(summary) == 0: return cls(propIDDict=propDict, @@ -584,7 +612,7 @@ def _read_summary_table_raw(engine, opsimVars, propIDs, subset): @staticmethod def _get_propIDs(tableNames, engine, opsimversion, subset, - user_propIDs=None): + user_propIDs=None, use_proposal_table=False): """return a sequence of `proposalId` which determine the subset of observations from tha `summary` table. @@ -607,14 +635,19 @@ def _get_propIDs(tableNames, engine, opsimversion, subset, """ # Read the proposal table to find out which propID corresponds to # the subsets requested - proposals = pd.read_sql_table(tableNames[1], con=engine) - propDict = OpSimOutput.get_propIDDict(proposals, opsimversion=opsimversion) + if use_proposal_table: + proposals = pd.read_sql_table(tableNames[1], con=engine) + propDict = OpSimOutput.get_propIDDict(proposals, + opsimversion=opsimversion) - # Seq of propIDs consistent with subset - _propIDs = OpSimOutput.propIDVals(subset, propDict, proposals) + # Seq of propIDs consistent with subset + _propIDs = OpSimOutput.propIDVals(subset, propDict, proposals) + + # If propIDs and subset were both provided, override subset propIDs + propIDs = OpSimOutput._overrideSubsetPropID(user_propIDs, _propIDs) + else: + propDict, propIDs, proposals = None, None, None - # If propIDs and subset were both provided, override subset propIDs - propIDs = OpSimOutput._overrideSubsetPropID(user_propIDs, _propIDs) return propDict, propIDs, proposals @@ -734,17 +767,20 @@ def _overrideSubsetPropID(propIDs, _propIDs): return propIDs @staticmethod - def get_allowed_subsets(opsimversion): + def get_allowed_subsets(opsimversion, use_proposal_table): """Provide a sequence of implemented subset values""" # Note this is really a version which has annotations on top of fbs v1p3 # Making this if statement superfluous + if not use_proposal_table: + return 'unique_all' + if opsimversion.lower() == 'fbsv1': return ('_all', 'ddf', 'wfd', 'combined', 'unique_all') else: return ('_all', 'ddf', 'wfd', 'combined', 'unique_all') @staticmethod - def get_propIDDict(proposalDF, opsimversion='fbsv2'): + def get_propIDDict(proposalDF, opsimversion='fbsv2', use_proposal_table=True): """ Return a dictionary with keys 'ddf', ad 'wfd' with the proposal IDs corresponding to deep drilling fields (ddf) and universal cadence (wfd) @@ -760,6 +796,9 @@ def get_propIDDict(proposalDF, opsimversion='fbsv2'): dictionary with keys 'wfd' and 'ddf' with values given by integers corresponding to propIDs for these proposals """ + + ### WHY DO WE NEED THIS ANYMORE ? + ### Is this not covered in opsimvarsiablesforversion Now? oss_wfdName = 'wfd' oss_ddfName = 'ddf' From c1e1c04f9dbd4c0b9ddb9bab42af64cf22389ca2 Mon Sep 17 00:00:00 2001 From: "R.Biswas" Date: Mon, 12 Dec 2022 15:25:19 +0100 Subject: [PATCH 05/13] WIP: Added a variable `use_proposal_table` which is by default `True` so that by default the previous behaviour is repeated. However when we use this variable set to `False`, the program ignores the Proposal Table. It also does not perform the de-duplication that was necessary in earlier versions of `OpSim` , but are not required anymore. # Please enter the commit message for your changes. Lines starting modified: ../opsimsummary/opsim_out.py --- opsimsummary/opsim_out.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/opsimsummary/opsim_out.py b/opsimsummary/opsim_out.py index b7ef2c4..e4951df 100644 --- a/opsimsummary/opsim_out.py +++ b/opsimsummary/opsim_out.py @@ -432,7 +432,7 @@ def fromOpSimDB(cls, dbname, user_propIDs=None, # dithercolumns=None, # add_dithers=False, - use_proposal_table=False, + use_proposal_table=True, tableNames=('Summary', 'Proposal'), filterNull=False, **kwargs): @@ -550,7 +550,7 @@ def fromOpSimDB(cls, dbname, # Drop Duplicates if subset != '_all': # Drop duplicates unless this is to write out the entire OpSim - summary = cls.dropDuplicates(summary, propDict, opsimversion) + summary = cls.dropDuplicates(summary, propDict, opsimversion, use_proposal_table=use_proposal_table) # Set Standard Index summary.set_index('obsHistID', inplace=True) @@ -612,7 +612,7 @@ def _read_summary_table_raw(engine, opsimVars, propIDs, subset): @staticmethod def _get_propIDs(tableNames, engine, opsimversion, subset, - user_propIDs=None, use_proposal_table=False): + user_propIDs=None, use_proposal_table=True): """return a sequence of `proposalId` which determine the subset of observations from tha `summary` table. @@ -662,7 +662,7 @@ def _get_sql_engine(dbname): return engine @staticmethod - def dropDuplicates(df, propIDDict, opsimversion): + def dropDuplicates(df, propIDDict, opsimversion, use_proposal_table=True): """ drop duplicates ensuring keeping identity of ddf visits @@ -675,6 +675,8 @@ def dropDuplicates(df, propIDDict, opsimversion): ------- `pd.DataFrame` with the correct propID and duplicates dropped """ + if not use_proposal_table: + return df if opsimversion == 'sstf': return df @@ -767,7 +769,7 @@ def _overrideSubsetPropID(propIDs, _propIDs): return propIDs @staticmethod - def get_allowed_subsets(opsimversion, use_proposal_table): + def get_allowed_subsets(opsimversion, use_proposal_table=True): """Provide a sequence of implemented subset values""" # Note this is really a version which has annotations on top of fbs v1p3 # Making this if statement superfluous From e854615c9e9a9774c41746ea9ecb123054b30b31 Mon Sep 17 00:00:00 2001 From: "R.Biswas" Date: Mon, 12 Dec 2022 15:34:37 +0100 Subject: [PATCH 06/13] WIP: A new script based on `make_simlibs.py` to be used explicitly ignoring proposal tables. new file: make_simlibs_no_proposal.py --- scripts/make_simlibs_no_proposal.py | 438 ++++++++++++++++++++++++++++ 1 file changed, 438 insertions(+) create mode 100755 scripts/make_simlibs_no_proposal.py diff --git a/scripts/make_simlibs_no_proposal.py b/scripts/make_simlibs_no_proposal.py new file mode 100755 index 0000000..b489bc2 --- /dev/null +++ b/scripts/make_simlibs_no_proposal.py @@ -0,0 +1,438 @@ +#!/usr/bin/env python +""" +Script to make Simlib files from OpSim DataBases (supported versions include +OpSim v3, OpSim v4, AltSched outputs. + To get usage : python make_simlibs.py -h +""" +import os +from argparse import ArgumentParser +import numpy as np +import pandas as pd +import healpy as hp +import opsimsummary as oss +from opsimsummary import Simlibs, OpSimOutput +import sys +import time +import datetime +import subprocess + + +def write_genericSimlib(simlibFilename, + summary, + minVisits, + maxVisits, + numFields, + footprint_hids=None, + footprint_hids_fname=None, + vetoed_hids=None, + author_name=None, + mapFile=None, + rng=np.random.RandomState(0), + mwebv=0., + raCol='ditheredRA', + decCol='ditheredDec', + angleUnit='degrees', + opsimversion='fbsv2', + indexCol='obsHistID', + nside=256, + fieldType='DDF', + #opsimoutput='minion_1016_sqlite.db', + #opsim_output='opsim_output', + script_name=None, + surveypix_file=None): + """ + Write out simlibs from a summary dataFrame + + Parameters + ---------- + simlibFilename : string + absolute path to simlib file to be written out + summary : `pd.dataFrame` + summary of observations + minVisits : int + minimum number of visits, such that fields with lower numbers of visits + over the survey are not considered. + maxVisits : int + maximum number of visits, such that fields with higher numbers of visits + over the survey are not considered. + numFields : int + number of sample fields requested in the simlib file. If this number + exceeds the number of fields satisfying the number of visits, all of the + fields satisfying the constraints are included in the simlib + author_name : string, defaults to None + Author name. If not provided, assumed to be None. The author name will be replaced by + user login. + vetoed_hids : set of integers, defaults to `None` + if not `None`, set of integers representing healpix Ids in the same + (`nest`|`ring`) which should not be used. If `None`, ignore. + opsim_output: str + name of the opsim_output 'opsim_output' + script_name : + surveypix_file : abspath to surveypix_file, defaults to None + If not None, uses this file to find the libids to simulate. Should + have an ordered dataframe of ra, dec, simlibIds + + Returns + ------- + surveypix : dataframe + healpixels in our selected footprint + surveydf : dataframe + selected visits with information needed for simlib files + + + Notes + ----- + Aside from the returns, this function also writes out the simlib files, and the surveypix_file if it was + constructed in the method. It also returns a set of logs to stdout. + """ + print('========================') + print(mapFile, surveypix_file) + + print('========================') + + opsimsummary_version = oss.__version__ + + minMJD = summary.expMJD.min() + maxMJD = summary.expMJD.max() + + simlibs = Simlibs(summary, + usePointingTree=True, + raCol=raCol, + decCol=decCol, + angleUnit=angleUnit, + indexCol=indexCol, + opsimversion=opsimversion) + + # This is a dataframe with healpix Ids that get between minVisits + # and maxVisits + + surveydf = simlibs.observedVisitsinRegion(minVisits=minVisits, + maxVisits=maxVisits, + writeFile=False, + nside=nside) + + # In SNANA, we do a simulation by sampling observed visits + # The sampling is done by choosing a spatial samples of the footprint + # which must be representative of the footprint, rather than things like edges. + # This selected footprint is described as a set of healpixels and is described by + # indices called hids in surveypix + + + # First collect hids of the footprint if available + + ## use sequence if provided + if footprint_hids is not None: + hids = set(footprint_hids) + print('Using passed sequence for hids\n') + elif footprint_hids_fname is not None: + surveyPix = pd.read_csv(footprint_hids_fname) + hids = set(surveyPix.index.values) + print('Obtaining hids from provided surveyPix file\n') + elif summary is not None: + hids = set(surveydf.query('numVisits >= @minVisits').index.values) + print('Constructing hids from simlibs from summary\n') + else: + raise ValueError('It does not make sense to not have a summary file \n') + + + if vetoed_hids is not None: + vetoed_hids = set(vetoed_hids) + avail = hids - vetoed_hids + else: + avail = hids + + surveydf = surveydf.loc[avail].query('numVisits >= @minVisits ') + surveyPix = simlibs.get_surveyPix(surveydf, numFields=numFields, rng=rng) + totalfields = len(surveydf) + + + #### # The surveypix may be constructed or read in from a file : `surveypix_file` + #### if surveypix_file is None: + #### # vetoed hids may describe parts of the footprint that are not part of the survey, e.g Deep Drilling + #### # Fields when you want to study Wide Fast Deep. + #### if vetoed_hids is not None: + #### hids = set(surveydf.index.values) + #### selected = hids - vetoed_hids + #### surveydf = surveydf.loc[selected] + #### totalfields = len(surveydf) + #### surveyPix = simlibs.get_surveyPix(surveydf, numFields=numFields, rng=rng) + #### else: + #### print('Reading in surveypix file\n') + #### print('You should only be using this if you are studying ToO proposals\n') + #### surveyPix = pd.read_csv(surveypix_file) + #### totalfields = len(surveyPix) + + fields = simlibs.simlibs_for_fields(surveyPix, mwebv=mwebv) + + # Area of our selected footprint in sq. deg + area = hp.nside2pixarea(nside, degrees=True) * np.float(totalfields) + + # Area of our selected footprint in solid angle (Used by SNANA for simulation) + solidangle = hp.nside2pixarea(nside, degrees=False) * np.float(totalfields) + + print('Going to write simlib file {0} for opsim output\n') + # This is not very reliable + if script_name is None: + script_name = 'OpSimSummary/scripts/make_simlibs.py' + dt = datetime.datetime.now() + ts = dt.isoformat() + + + # Author name. Will be entered in SNANA Simlib documentation. + if author_name is None: + author_name = os.getlogin() + + ## git information + try : + label = subprocess.check_output(['git', 'describe', '--dirty']).strip().decode('utf-8') + # Assume that there will be git versions before and after 2.22 + branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip().decode('utf-8') + if label.endswith('dirty'): + git_sha = label.split('-dirty')[0] + dirty = 'dirty' + else: + git_sha = label + dirty = '' + + git_info = f'The script was part of a {dirty} git repository in the branch {branch} and commit SHA {git_sha} and was labelled with a version : {oss.__version__}' + except: + git_info = 'The script was not detected to be a part of a git repository' + + + # OpSimSummary version information + version = oss.__version__ # RB: I think this line is not needed + + comment = 'DOCUMENTATION:\n' + comment += f' PURPOSE: simulate LSST based on mock opsim version {opsim_output}\n' + comment += ' INTENT: Nominal\n' + comment += ' USAGE_KEY: SIMLIB_FILE\n' + comment += ' USAGE_CODE: snlc_sim.exe\n' + comment += ' VALIDATION_SCIENCE: \n' + comment += f' FIELD: {fieldType}\n' + comment += ' NOTES: \n' + comment += ' PARAMS MINMJD: {:.4f}\n'.format(minMJD) + comment += ' PARAMS MAXMJD: {:.4f}\n'.format(maxMJD) + comment += ' PARAMS TOTAL_AREA: {:.3f}\n'.format(area) + comment += ' PARAMS SOLID_ANGLE: {:.3f}\n'.format(solidangle) + comment += ' VERSIONS:\n' + comment += f' - DATE : {dt.strftime(format="%y-%m-%d")}\n' + comment += f' AUTHORS : {author_name}, OpSimSummary version {oss.__version__}\n' + comment += 'DOCUMENTATION_END:\n' + + doc = comment + + # Additional commments which can be used as keys, or as additional documentation + # SKIP + #### comment = 'COMMENT: Total area corresponding to this simlib is {0:.1f} sq degrees or a solid angle of {1:4f} \n'.format(area, solidangle) + #### comment += 'COMMENT: This is a simlib corresponding to {0} in the OpSim Output {1} using the script {3} in version {2} at time {4}. {5}\n'.format( + #### fieldType, + #### opsimoutput, + #### opsimsummary_version, + #### script_name, + #### ts, + #### git_info) + #### comment += '\nCOMMENT: PARAMS MINMJD: {:.4f}\n'.format(minMJD) + #### comment += 'COMMENT: PARAMS MAXMJD: {:.4f}\n'.format(maxMJD) + #### comment += 'COMMENT: PARAMS TOTAL_AREA: {:.3f}\n'.format(area) + #### comment += 'COMMENT: PARAMS SOLID_ANGLE: {:.3f}\n'.format(solidangle) + + simlibs.writeSimlib(simlibFilename, + fields, + mwebv=mwebv, + doc=doc, + comments='\n', + numLibId=numFields) + surveyPix = surveyPix.reset_index().query('simlibId > -1').set_index('simlibId') + surveyPix = surveyPix.reset_index().sort_values(by='simlibId').set_index('simlibId') + + # Write out the surveyPix + mapFile = 'testing' + fieldType + surveyPix.to_csv(mapFile) + return surveyPix, surveydf + +if __name__ == '__main__': + parser = ArgumentParser(description='write out simlibs from an OpSim Database') + parser.add_argument('--data_root', help='absolute path to data directory containing dither files and opsim database, defaults to "/"', + default='/') + parser.add_argument('--dbname', help='path to sqlite database output from OpSim relative to data_root, defaults to `None`, which will crash', default=None) + parser.add_argument('--opsim_output', help='Optionally required for non-standard/non-Rubin opsim output developments: name of the opsim output being used, will be obtained from filename if skipped', default=None, type=str) + parser.add_argument('--no_write_ddf_simlib', help='Whether to write out DDF simlib', + dest='write_ddf_simlib', action='store_false') + parser.add_argument('--no_get_ddf_pixels', help='Whether to obtain ddf pixels from opsim or use null set', + dest='No_get_ddf_pixels', action='store_false') + parser.add_argument('--no_write_wfd_simlib', help='Whether to write out WFD simlib, defaults to writing it out', + dest='write_wfd_simlib', action='store_false') + parser.add_argument('--opsimversion', help='version of opsim used lsstv3|lsstv4, defaults to lsstv3', + default='lsstv3') + parser.add_argument('--summaryTableName', help='name of Summary Table Summary|SummaryAllProps, defaults to "Summary"', + default='Summary') + parser.add_argument('--ddf_surveypix_file', help='absolute path to DDF surveypix_file, defaults to `None`', + default=None) + parser.add_argument('--wfd_surveypix_file', help='absolute path to WFD surveypix_file, defaults to `None`', + default=None) + parser.add_argument('--ddf_simlibfilename', help='absolute path to DDF simlib file to write out, defaults to `None`', + default=None) + parser.add_argument('--wfd_simlibfilename', help='absolute path to WFD simlib file to write out, defaults to `None`', + default=None) + parser.add_argument('--numFields_DDF', help='number of locations in DDF where simlib fields are located, defaults to 133', + default=133, type=int) + parser.add_argument('--numFields_WFD', help='number of locations in DDF where simlib fields are located, defaults to 50000', + default=50000, type=int) + parser.add_argument('--filterNull', help='if added, then the summary table of the OpSim file will be filtered of rows that appear to have null values', + dest='filt_Null', action='store_true') + parser.add_argument('--authorName', help='Name of the author to be recorded in documentation. If skipped, the login will be used instead', type=str, default=None) + + print("read in command line options and figuring out what to do\n") + print("---------------------------------------------------------\n") + print("We are using opsimsummary version {0}, the library is located at {1}".format(oss.__version__, oss.__file__)) + print("The path is {}".format(sys.path)) + print("The python version is {}".format(sys.version)) + print("---------------------------------------------------------\n") + sys.stdout.flush() + args = parser.parse_args() + + print("---------------------------------------------------------\n") + print("Obtain arguments\n") + print("---------------------------------------------------------\n") + if args.dbname is None: + print('You have to supply `dbname` as the absolute path to the OpSim Output. \n') + print('Alternatively `dbname` can also be a filename, relative to `data_root` which must be specified as well. \n') + raise ValueError('`dbname` cannot be `None`') + + data_root = args.data_root + author_name = args.authorName + dbname = os.path.join(data_root, args.dbname) + basename = dbname.split('/')[-1].split('.db')[0] + + opsim_output = args.opsim_output + if opsim_output is None: + opsim_output = basename + + filternulls = False + if args.filt_Null : + print('filtering the raw summary table, with this option, null values in `fiveSigmaDepth` will be skipped') + print("We do not recommend using this option unless you are sure you want this\n") + filternulls = True + + get_ddf_pixels = False + print("get_ddf_pixels", get_ddf_pixels) + + + print("\n\n Task: Obtaining pointing location \n") + sys.stdout.flush() + summaryTableName = args.summaryTableName + opsimoutput = os.path.basename(dbname) + opsimversion = args.opsimversion + write_wfd_simlib = args.write_wfd_simlib + write_ddf_simlib = False + wfd_simlibfilename = args.wfd_simlibfilename + ddf_simlibfilename = False + + if wfd_simlibfilename is None: + wfd_simlibfilename = basename +'_all.simlib' + availwfdFileName = basename + "_all_avail.csv" + selectedwfdFileName = basename + "_all_sel.csv" + print('output file names for all are {0}, {1}, {2}'.format(wfd_simlibfilename, availwfdFileName, selectedwfdFileName)) + + numFields_WFD = args.numFields_WFD + print(args) + + sys.stdout.flush() + + # find ddf healpixels if the `get_ddf_pixels` is True and + # store in ddf_hid + # Else: ddf_hid is an empty set + get_ddf_pixels = False + if get_ddf_pixels: + print('Finding the DDF healpixels \n') + # read in opsim with only DDF observations + opsout_ddf = OpSimOutput.fromOpSimDB(dbname, opsimversion=opsimversion, + subset='ddf', + filterNull=filternulls) + + + if len(opsout_ddf.summary) > 0: + print("writing out ddf pixels\n") + simlib_ddf = Simlibs(opsout_ddf.summary, opsimversion=opsimversion, + usePointingTree=True) + # These are the ddf healpix pixels + ddf_hid = set(simlib_ddf.observedVisitsinRegion().index.values) + else: + + print("writing out null set of ddf pixels\n") + ddf_hid = set([]) + print("written out null set of ddf pixels\n") + + + print('There are {} pixels in the ddf fields'.format(len(ddf_hid))) + # read the database into a `pd.DataFrame` + tstart = time.time() + print("\n\n Task: reading database {0} at time {1}. This can take a while ... ".format(dbname, tstart)) + sys.stdout.flush() + + # read with subset = 'combined', This should have WFD + DDF pointings and reads in the entire opsim + opsout = OpSimOutput.fromOpSimDB(dbname, + opsimversion=opsimversion, + tableNames=(summaryTableName, 'Proposal'), + subset='unique_all', + filterNull=False, + use_proposal_table=False) + tend = time.time() + print("finished reading database {0} at time {1}".format(dbname, tend)) + print("reading the db took {} minutes".format((tend-tstart)/60.0)) + sys.stdout.flush() + + # Summary table with this set. + summary = opsout.summary + script_name = os.path.abspath(__file__) + print('========================') + print( + '\n dbname', dbname, + '\n script_name', script_name, + '\n author_name', author_name, + '\n opsim_output', opsim_output) + print('========================') + + + write_ddf_simlib = False + if write_ddf_simlib: + print('\n\n Task: writing out simlib for DDF') + # 133 random locations is similar density of locations in WFD. + x, y = write_genericSimlib(simlibFilename=ddf_simlibfilename, + summary=summary, + minVisits=500, maxVisits=None, + numFields=numFields_DDF, + mapFile=None, + fieldType='DDF', + # opsimoutput=dbname, + script_name=script_name, + author_name=author_name, + # opsim_output=opsim_output, + # surveypix_file=args.ddf_surveypix_file) + footprint_hids=ddf_hid) + print('Finished writing out simlib for DDF') + print('\n\n Task: write mapping to csv') + x.to_csv(selectedddfFileName) + y.to_csv(availddfFileName) + print('Finished writing mapping to csv') + sys.stdout.flush() + sys.stdout.flush() + + + if write_wfd_simlib : + print('\n\n Task: writing out simlib for WFD') + sys.stdout.flush() + x, y = write_genericSimlib(simlibFilename=wfd_simlibfilename, + summary=summary, minVisits=500, maxVisits=100000, + numFields=numFields_WFD, mapFile='wfd_minion_1016_sqlite.csv', + fieldType='WFD', # opsimoutput=dbname, + vetoed_hids=ddf_hid, + script_name=script_name) + print('Finished writing out simlib for WFD') + print('\n\n Task: write mapping to csv') + x.to_csv(selectedwfdFileName) + y.to_csv(availwfdFileName) + print('Finished writing mapping to csv') + sys.stdout.flush() + print('finished job') + sys.stdout.flush() From 20639b186f071fcd55a2aabce5bb5bfaea76e835 Mon Sep 17 00:00:00 2001 From: "Rob Knop (Nersc)" Date: Wed, 30 Aug 2023 13:29:20 -0700 Subject: [PATCH 07/13] In make_simlibs_no_proposal.py, fix a pandas problem (can't index by set, use a list), get rid of a couple of numpy deprecation warnings, and make the default opsimsummaryversion fbsv2 --- scripts/make_simlibs_no_proposal.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/make_simlibs_no_proposal.py b/scripts/make_simlibs_no_proposal.py index b489bc2..6d8bd25 100755 --- a/scripts/make_simlibs_no_proposal.py +++ b/scripts/make_simlibs_no_proposal.py @@ -140,8 +140,8 @@ def write_genericSimlib(simlibFilename, avail = hids - vetoed_hids else: avail = hids - - surveydf = surveydf.loc[avail].query('numVisits >= @minVisits ') + + surveydf = surveydf.loc[list(avail)].query('numVisits >= @minVisits ') surveyPix = simlibs.get_surveyPix(surveydf, numFields=numFields, rng=rng) totalfields = len(surveydf) @@ -165,10 +165,10 @@ def write_genericSimlib(simlibFilename, fields = simlibs.simlibs_for_fields(surveyPix, mwebv=mwebv) # Area of our selected footprint in sq. deg - area = hp.nside2pixarea(nside, degrees=True) * np.float(totalfields) + area = hp.nside2pixarea(nside, degrees=True) * np.float64(totalfields) # Area of our selected footprint in solid angle (Used by SNANA for simulation) - solidangle = hp.nside2pixarea(nside, degrees=False) * np.float(totalfields) + solidangle = hp.nside2pixarea(nside, degrees=False) * np.float64(totalfields) print('Going to write simlib file {0} for opsim output\n') # This is not very reliable @@ -262,8 +262,8 @@ def write_genericSimlib(simlibFilename, dest='No_get_ddf_pixels', action='store_false') parser.add_argument('--no_write_wfd_simlib', help='Whether to write out WFD simlib, defaults to writing it out', dest='write_wfd_simlib', action='store_false') - parser.add_argument('--opsimversion', help='version of opsim used lsstv3|lsstv4, defaults to lsstv3', - default='lsstv3') + parser.add_argument('--opsimversion', help='version of opsim used lsstv3|lsstv4|, defaults to fbsv2', + default='fbsv2') parser.add_argument('--summaryTableName', help='name of Summary Table Summary|SummaryAllProps, defaults to "Summary"', default='Summary') parser.add_argument('--ddf_surveypix_file', help='absolute path to DDF surveypix_file, defaults to `None`', From 409cac601d572a62a569c25a3e77bb420a5dd9c8 Mon Sep 17 00:00:00 2001 From: "Rob Knop (Nersc)" Date: Thu, 31 Aug 2023 08:53:00 -0700 Subject: [PATCH 08/13] Get hostname from os.getenv( "HOSTNAME" ) instead of a subprocess to avoid a b'' in the output; rename DECL to DEC as per current SNANA standard --- opsimsummary/simlib.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/opsimsummary/simlib.py b/opsimsummary/simlib.py index e52fbd8..b21f564 100644 --- a/opsimsummary/simlib.py +++ b/opsimsummary/simlib.py @@ -113,8 +113,7 @@ def get_simlibVars(user=None, host=None, pixelSize=0.2, telescope='LSST', # report a host on which the calculations are done. either from # constructor parameters or from the system hostname utility if host is None: - proc = subprocess.Popen('hostname', stdout=subprocess.PIPE) - host, err = proc.communicate() + host = os.getenv( 'HOSTNAME' ) x = (('user', user), ('host', host), ('pixelSize', pixelSize), @@ -257,7 +256,7 @@ def fieldheader(self, fieldID, ra, dec, opsimtable, mwebv=0.0, s += 'LIBID: {0:10d}'.format(fieldID) +'\n' if fieldtype is not None: s += 'Field: {}\n'.format(fieldtype) - tmp = 'RA: {0:+10.6f} DECL: {1:+10.6f} NOBS: {2:10d} MWEBV: {3:5.2f}' + tmp = 'RA: {0:+10.6f} DEC: {1:+10.6f} NOBS: {2:10d} MWEBV: {3:5.2f}' tmp += ' PIXSIZE: {4:5.3f}' s += tmp.format(ra, dec, nobs, mwebv, self.pixelSize) + '\n' # s += 'LIBID: {0:10d}'.format(fieldID) + '\n' @@ -653,7 +652,7 @@ class FieldSimlib(object): meta : dict metadata associated with the field, which has at least the following keys: - LIBID, RA, DECL, MWEBV, NOBS, PIXSIZE + LIBID, RA, DEC, MWEBV, NOBS, PIXSIZE data : `~pd.DataFrame` object with the observations and having at least the following columns: 'MJD', 'IDEXPT', 'FLT', 'GAIN', 'NOISE', 'SKYSIG', 'PSF1', 'PSF2', 'PSFRatio', 'ZPTAVG', 'ZPTERR', 'MAG']. The meanings of From 949d6b9f9ce4bc471295fa53626f36af6b175fe4 Mon Sep 17 00:00:00 2001 From: "Rob Knop (Nersc)" Date: Tue, 19 Dec 2023 06:00:41 -0800 Subject: [PATCH 09/13] x --- scripts/make_simlibs_no_proposal.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/make_simlibs_no_proposal.py b/scripts/make_simlibs_no_proposal.py index 6d8bd25..45c60fe 100755 --- a/scripts/make_simlibs_no_proposal.py +++ b/scripts/make_simlibs_no_proposal.py @@ -87,9 +87,9 @@ def write_genericSimlib(simlibFilename, """ print('========================') print(mapFile, surveypix_file) - + print('========================') - + opsimsummary_version = oss.__version__ minMJD = summary.expMJD.min() @@ -169,7 +169,7 @@ def write_genericSimlib(simlibFilename, # Area of our selected footprint in solid angle (Used by SNANA for simulation) solidangle = hp.nside2pixarea(nside, degrees=False) * np.float64(totalfields) - + print('Going to write simlib file {0} for opsim output\n') # This is not very reliable if script_name is None: @@ -197,8 +197,8 @@ def write_genericSimlib(simlibFilename, git_info = f'The script was part of a {dirty} git repository in the branch {branch} and commit SHA {git_sha} and was labelled with a version : {oss.__version__}' except: git_info = 'The script was not detected to be a part of a git repository' - - + + # OpSimSummary version information version = oss.__version__ # RB: I think this line is not needed @@ -262,7 +262,7 @@ def write_genericSimlib(simlibFilename, dest='No_get_ddf_pixels', action='store_false') parser.add_argument('--no_write_wfd_simlib', help='Whether to write out WFD simlib, defaults to writing it out', dest='write_wfd_simlib', action='store_false') - parser.add_argument('--opsimversion', help='version of opsim used lsstv3|lsstv4|, defaults to fbsv2', + parser.add_argument('--opsimversion', help='version of opsim used lsstv3|lsstv4|fbsv2, defaults to fbsv2', default='fbsv2') parser.add_argument('--summaryTableName', help='name of Summary Table Summary|SummaryAllProps, defaults to "Summary"', default='Summary') @@ -281,7 +281,7 @@ def write_genericSimlib(simlibFilename, parser.add_argument('--filterNull', help='if added, then the summary table of the OpSim file will be filtered of rows that appear to have null values', dest='filt_Null', action='store_true') parser.add_argument('--authorName', help='Name of the author to be recorded in documentation. If skipped, the login will be used instead', type=str, default=None) - + print("read in command line options and figuring out what to do\n") print("---------------------------------------------------------\n") print("We are using opsimsummary version {0}, the library is located at {1}".format(oss.__version__, oss.__file__)) @@ -290,7 +290,7 @@ def write_genericSimlib(simlibFilename, print("---------------------------------------------------------\n") sys.stdout.flush() args = parser.parse_args() - + print("---------------------------------------------------------\n") print("Obtain arguments\n") print("---------------------------------------------------------\n") @@ -327,7 +327,7 @@ def write_genericSimlib(simlibFilename, write_ddf_simlib = False wfd_simlibfilename = args.wfd_simlibfilename ddf_simlibfilename = False - + if wfd_simlibfilename is None: wfd_simlibfilename = basename +'_all.simlib' availwfdFileName = basename + "_all_avail.csv" @@ -336,7 +336,7 @@ def write_genericSimlib(simlibFilename, numFields_WFD = args.numFields_WFD print(args) - + sys.stdout.flush() # find ddf healpixels if the `get_ddf_pixels` is True and @@ -392,7 +392,7 @@ def write_genericSimlib(simlibFilename, '\n author_name', author_name, '\n opsim_output', opsim_output) print('========================') - + write_ddf_simlib = False if write_ddf_simlib: From 40b2d758b530ef224619a3dbdff5193d16704a80 Mon Sep 17 00:00:00 2001 From: "Rob Knop (Nersc)" Date: Thu, 18 Jul 2024 12:57:33 -0700 Subject: [PATCH 10/13] Add FIELD in the simlib documentation with WFD or DDF --- scripts/make_simlibs_no_proposal.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/scripts/make_simlibs_no_proposal.py b/scripts/make_simlibs_no_proposal.py index 45c60fe..2fbd10e 100755 --- a/scripts/make_simlibs_no_proposal.py +++ b/scripts/make_simlibs_no_proposal.py @@ -202,19 +202,21 @@ def write_genericSimlib(simlibFilename, # OpSimSummary version information version = oss.__version__ # RB: I think this line is not needed - comment = 'DOCUMENTATION:\n' + comment = f'DOCUMENTATION:\n' comment += f' PURPOSE: simulate LSST based on mock opsim version {opsim_output}\n' - comment += ' INTENT: Nominal\n' - comment += ' USAGE_KEY: SIMLIB_FILE\n' - comment += ' USAGE_CODE: snlc_sim.exe\n' - comment += ' VALIDATION_SCIENCE: \n' + comment += f' INTENT: Nominal\n' + comment += f' USAGE_KEY: SIMLIB_FILE\n' + comment += f' USAGE_CODE: snlc_sim.exe\n' + comment += f' VALIDATION_SCIENCE: \n' comment += f' FIELD: {fieldType}\n' - comment += ' NOTES: \n' - comment += ' PARAMS MINMJD: {:.4f}\n'.format(minMJD) - comment += ' PARAMS MAXMJD: {:.4f}\n'.format(maxMJD) - comment += ' PARAMS TOTAL_AREA: {:.3f}\n'.format(area) - comment += ' PARAMS SOLID_ANGLE: {:.3f}\n'.format(solidangle) - comment += ' VERSIONS:\n' + comment += f' NOTES: \n' + comment += f' PARAMS MINMJD: {minMJD:.4f}\n' + comment += f' PARAMS MAXMJD: {maxMJD:.4f}\n' + comment += f' PARAMS TOTAL_AREA: {area:.3f}\n' + comment += f' PARAMS SOLID_ANGLE: {solidangle:.3f}\n' + comment += f' WARNING_NOTICE: DDF approximately determined by nobs>=1100\n' + comment += f' COMMAND: {sys.argv}\n' + comment += f' VERSIONS:\n' comment += f' - DATE : {dt.strftime(format="%y-%m-%d")}\n' comment += f' AUTHORS : {author_name}, OpSimSummary version {oss.__version__}\n' comment += 'DOCUMENTATION_END:\n' From f3db5046ea51010c50d7ed2763d8f1ac81af915f Mon Sep 17 00:00:00 2001 From: "Rob Knop (Nersc)" Date: Thu, 18 Jul 2024 12:59:50 -0700 Subject: [PATCH 11/13] left out of last commit --- opsimsummary/simlib.py | 13 ++++++++----- opsimsummary/summarize_opsim.py | 4 ++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/opsimsummary/simlib.py b/opsimsummary/simlib.py index b21f564..7a7bdf9 100644 --- a/opsimsummary/simlib.py +++ b/opsimsummary/simlib.py @@ -230,7 +230,7 @@ def add_simlibCols(opsimtable, pixelSize=0.2): return opsimtable def fieldheader(self, fieldID, ra, dec, opsimtable, mwebv=0.0, - fieldtype=None): + fieldtype=None, nobs_min_ddf=1100): """ Parameters ---------- @@ -256,8 +256,12 @@ def fieldheader(self, fieldID, ra, dec, opsimtable, mwebv=0.0, s += 'LIBID: {0:10d}'.format(fieldID) +'\n' if fieldtype is not None: s += 'Field: {}\n'.format(fieldtype) - tmp = 'RA: {0:+10.6f} DEC: {1:+10.6f} NOBS: {2:10d} MWEBV: {3:5.2f}' - tmp += ' PIXSIZE: {4:5.3f}' + tmp = 'RA: {0:+10.6f} DEC: {1:+10.6f} NOBS: {2:10d} MWEBV: {3:5.2f}' + tmp += ' PIXSIZE: {4:5.3f}' + # RKNOP 2024-07-17 : added the following line; the default + # nobs_min_ddf cutoff (1100) was based on looking at a histogram + # of nobs from baseline 3.4 + tmp += f' FIELD: {"DDF" if nobs>=nobs_min_ddf else "WFD"}' s += tmp.format(ra, dec, nobs, mwebv, self.pixelSize) + '\n' # s += 'LIBID: {0:10d}'.format(fieldID) + '\n' s += '# CCD CCD PSF1 PSF2 PSF2/1' +'\n' @@ -337,8 +341,7 @@ def simLibheader(self, numLibId=None, saturation_flag=1024, # of the logic for other filter names. so ducking for now s = doc s += '\n\n\n' - s += 'SURVEY: {0:} FILTERS: ugrizY TELESCOPE: {1:}\n'.format(survey, - telescope) + s += 'SURVEY: {0:} FILTERS: ugrizY\n'.format(survey) s += 'USER: {0:} HOST: {1}\n'.format(user, host) if numLibId is not None: s += 'NLIBID: {}\n'.format(numLibId) diff --git a/opsimsummary/summarize_opsim.py b/opsimsummary/summarize_opsim.py index 727ee7b..0070c75 100644 --- a/opsimsummary/summarize_opsim.py +++ b/opsimsummary/summarize_opsim.py @@ -397,7 +397,7 @@ def __init__(self, # tree queries # Keep mapping from integer indices to obsHistID - pointings.loc[:, 'intindex'] = np.arange(len(pointings)).astype(np.int) + pointings.loc[:, 'intindex'] = np.arange(len(pointings)).astype(int) self.indMapping = pointings['intindex'].reset_index().set_index('intindex') # Build Tree @@ -1013,7 +1013,7 @@ def writeSimLibField(self, fieldID): def simLibheader(self): #, user=None, host=None, survey='LSST', telescope='LSST'): # comment: I would like to generalize ugrizY to a sort but am not sure # of the logic for other filter names. so ducking for now - s = 'SURVEY: {0:} FILTERS: ugrizY TELESCOPE: {1:}\n'.format(self.survey, self.telescope) + s = 'SURVEY: {0:} FILTERS: ugrizY\n'.format(self.survey) s += 'USER: {0:} HOST: {1:}\n'.format(self.user, self.host) s += 'BEGIN LIBGEN\n' return s From 5e4035d9329c2247e96cb1c69fc37512c45fb611 Mon Sep 17 00:00:00 2001 From: Richard Kessler Date: Fri, 2 Aug 2024 12:09:08 -0700 Subject: [PATCH 12/13] add DDF only feature and write more to SIMLIB DOCANA block --- opsimsummary/simlib.py | 34 ++++++++++++++++++++++++----- scripts/make_simlibs_no_proposal.py | 13 ++++++++--- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/opsimsummary/simlib.py b/opsimsummary/simlib.py index 7a7bdf9..0ae7a8d 100644 --- a/opsimsummary/simlib.py +++ b/opsimsummary/simlib.py @@ -5,7 +5,7 @@ """ from __future__ import division, print_function, unicode_literals __all__ = ['SimlibMixin', 'Simlibs'] -import os +import os, sys import numpy as np import subprocess from io import StringIO, BytesIO @@ -250,7 +250,10 @@ def fieldheader(self, fieldID, ra, dec, opsimtable, mwebv=0.0, string used to construct `Field: fieldtype` line, if None this line is left out. """ - nobs = len(opsimtable) + nobs = len(opsimtable) + fieldname = "DDF" if nobs>=nobs_min_ddf else "WFD" # R.Kessler + if fieldname == "WFD" : return None + # String formatting s = '# --------------------------------------------' +'\n' s += 'LIBID: {0:10d}'.format(fieldID) +'\n' @@ -261,7 +264,7 @@ def fieldheader(self, fieldID, ra, dec, opsimtable, mwebv=0.0, # RKNOP 2024-07-17 : added the following line; the default # nobs_min_ddf cutoff (1100) was based on looking at a histogram # of nobs from baseline 3.4 - tmp += f' FIELD: {"DDF" if nobs>=nobs_min_ddf else "WFD"}' + tmp += f' FIELD: {fieldname}' s += tmp.format(ra, dec, nobs, mwebv, self.pixelSize) + '\n' # s += 'LIBID: {0:10d}'.format(fieldID) + '\n' s += '# CCD CCD PSF1 PSF2 PSF2/1' +'\n' @@ -306,6 +309,10 @@ def simlibFieldasString(self, fh, fieldID, ra, dec, opsimtable, # Write out the header for each field s = self.fieldheader(fieldID, ra, dec, opsimtable, mwebv=mwebv, fieldtype=fieldtype) + + if s is None: + return "" # R.Kessler + # Write out the actual field s += self.formatSimLibField(fieldID, opsimtable, sep=' ') # Write out the footer for each field @@ -366,7 +373,8 @@ def writeSimlib(self, filename, fields, doc='\n', comments='\n', # Write out the header to the simlib file simlib_header = self.simLibheader(numLibId=numLibId, doc=doc, comments=comments) fh.write(simlib_header) - + fh.flush() # RK + # Now write the actual simlib data to file for field in fields: @@ -381,6 +389,7 @@ def writeSimlib(self, filename, fields, doc='\n', comments='\n', opsimtable, mwebv=mwebv, fieldtype=fieldtype)) + fh.flush() # RK # Write out the header for each field # fh.write(self.fieldheader(num_fields, ra, dec, opsimtable, # mwebv=mwebv)) @@ -459,8 +468,9 @@ def get_surveyPix(self, surveydf, numFields=15, rng=np.random.RandomState(0)): mapping from the new index `simlibId` to the old index `hid` """ surveydf['simlibId'] = -1 - - if numFields <= len(surveydf): + len_surveydf = len(surveydf) + + if numFields <= len_surveydf : surveydf = surveydf.sample(n=numFields, replace=False, random_state=rng) # hids = rng.choice(surveydf.reset_index()['hid'].values, size=numFields, @@ -470,9 +480,21 @@ def get_surveyPix(self, surveydf, numFields=15, rng=np.random.RandomState(0)): random_state=rng) print("Warning: You have asked for more samples than the original number of fields") print('Printing original number of fields instead') + sys.stdout.flush() + hids = surveydf.reset_index()['hid'].values + # xxxxxxxxxxx + n_hids = len(hids) # xxxx .xyz + hids_unique = list(set(hids)) + n_unique = len(hids_unique) + + print(f"\n xxx len_surveydf={len_surveydf} n_hids={n_hids} n_unique={n_unique} " \ + f"\n xxx hids = \n{hids}\n") + sys.stdout.flush() + # xxxxxxxxxxx + surveydf.reset_index().set_index('hid') surveydf.loc[hids, 'simlibId'] = np.arange(len(hids)) return surveydf diff --git a/scripts/make_simlibs_no_proposal.py b/scripts/make_simlibs_no_proposal.py index 2fbd10e..82c1836 100755 --- a/scripts/make_simlibs_no_proposal.py +++ b/scripts/make_simlibs_no_proposal.py @@ -202,20 +202,24 @@ def write_genericSimlib(simlibFilename, # OpSimSummary version information version = oss.__version__ # RB: I think this line is not needed + COMMAND = ' '.join(sys.argv) # RK^2 comment = f'DOCUMENTATION:\n' comment += f' PURPOSE: simulate LSST based on mock opsim version {opsim_output}\n' comment += f' INTENT: Nominal\n' comment += f' USAGE_KEY: SIMLIB_FILE\n' comment += f' USAGE_CODE: snlc_sim.exe\n' comment += f' VALIDATION_SCIENCE: \n' - comment += f' FIELD: {fieldType}\n' + + # xxx mark comment += f' FIELD: {fieldType}\n' # obsolete logic + comment += f' FIELD: WFD+DDF\n' # R.Kessler Jul 2024: no more WFD or DDF only options + comment += f' NOTES: \n' comment += f' PARAMS MINMJD: {minMJD:.4f}\n' comment += f' PARAMS MAXMJD: {maxMJD:.4f}\n' comment += f' PARAMS TOTAL_AREA: {area:.3f}\n' comment += f' PARAMS SOLID_ANGLE: {solidangle:.3f}\n' comment += f' WARNING_NOTICE: DDF approximately determined by nobs>=1100\n' - comment += f' COMMAND: {sys.argv}\n' + comment += f' COMMAND: {COMMAND}\n' comment += f' VERSIONS:\n' comment += f' - DATE : {dt.strftime(format="%y-%m-%d")}\n' comment += f' AUTHORS : {author_name}, OpSimSummary version {oss.__version__}\n' @@ -424,11 +428,14 @@ def write_genericSimlib(simlibFilename, if write_wfd_simlib : print('\n\n Task: writing out simlib for WFD') sys.stdout.flush() + # R.Kessler July 2024 increase maxVists from 100k to 300k, and add nside=512 arg + # (replace previous default of nside=256) x, y = write_genericSimlib(simlibFilename=wfd_simlibfilename, - summary=summary, minVisits=500, maxVisits=100000, + summary=summary, minVisits=500, maxVisits=300000, numFields=numFields_WFD, mapFile='wfd_minion_1016_sqlite.csv', fieldType='WFD', # opsimoutput=dbname, vetoed_hids=ddf_hid, + nside=512, # R.Kessler July 2024 script_name=script_name) print('Finished writing out simlib for WFD') print('\n\n Task: write mapping to csv') From 39dbdda87d6dd4897d8645d79927e1537cb3e97e Mon Sep 17 00:00:00 2001 From: "Rob Knop (Nersc)" Date: Fri, 25 Oct 2024 11:48:37 -0700 Subject: [PATCH 13/13] fixes --- opsimsummary/simlib.py | 68 ++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/opsimsummary/simlib.py b/opsimsummary/simlib.py index 0ae7a8d..a57de6a 100644 --- a/opsimsummary/simlib.py +++ b/opsimsummary/simlib.py @@ -47,7 +47,7 @@ class SimlibMixin(object): - user (default can be None) - host (default can be None) - telescope - - survey + - survey - pixelSize In order to be able to write out the simlibs to disk, it should also have a method to provide a sequence (may be a generator) of fields, and @@ -111,7 +111,7 @@ def get_simlibVars(user=None, host=None, pixelSize=0.2, telescope='LSST', user = os.getlogin() # report a host on which the calculations are done. either from - # constructor parameters or from the system hostname utility + # constructor parameters or from the system hostname utility if host is None: host = os.getenv( 'HOSTNAME' ) x = (('user', user), @@ -167,7 +167,7 @@ def add_simlibCols(opsimtable, pixelSize=0.2): Returns ------- DataFrame with additional columns of 'simLibPsf', 'simLibZPTAVG', and - 'simLibSkySig' + 'simLibSkySig' .. note :: This was written from a piece of f77 code by David Cinabro sent by email on May 26, 2015. @@ -219,7 +219,7 @@ def add_simlibCols(opsimtable, pixelSize=0.2): zpt_cor = 2.5 * np.log10(1.0 + 1.0 / (area * tmp)) simlib_zptavg = zpt_approx + zpt_cor - # ZERO PT CALCULATION + # ZERO PT CALCULATION opsimtable['simLibZPTAVG'] = simlib_zptavg @@ -252,10 +252,13 @@ def fieldheader(self, fieldID, ra, dec, opsimtable, mwebv=0.0, """ nobs = len(opsimtable) fieldname = "DDF" if nobs>=nobs_min_ddf else "WFD" # R.Kessler - if fieldname == "WFD" : return None - + + # RK² NEED A FLAG so that we can do only-DDF simlibs; for now + # uncomment the next line to accomplish that. + # if fieldname == "WFD" : return None + # String formatting - s = '# --------------------------------------------' +'\n' + s = '# --------------------------------------------' +'\n' s += 'LIBID: {0:10d}'.format(fieldID) +'\n' if fieldtype is not None: s += 'Field: {}\n'.format(fieldtype) @@ -295,12 +298,12 @@ def formatSimLibField(self, fieldID, opsimtable, sep=' '): "{0:4.2f}".format(0.), # PSF2 "{0:4.3f}".format(0.), # PSFRatio "{0:6.2f}".format(data.simLibZPTAVG), # ZPTAVG - "{0:6.3f}".format(0.005), # ZPTNoise + "{0:6.3f}".format(0.005), # ZPTNoise "{0:+7.3f}".format(-99.)] # MAG s = sep.join(lst) y += s + '\n' return y - + def simlibFieldasString(self, fh, fieldID, ra, dec, opsimtable, mwebv=0.0, fieldtype=None): @@ -312,7 +315,7 @@ def simlibFieldasString(self, fh, fieldID, ra, dec, opsimtable, if s is None: return "" # R.Kessler - + # Write out the actual field s += self.formatSimLibField(fieldID, opsimtable, sep=' ') # Write out the footer for each field @@ -349,7 +352,7 @@ def simLibheader(self, numLibId=None, saturation_flag=1024, s = doc s += '\n\n\n' s += 'SURVEY: {0:} FILTERS: ugrizY\n'.format(survey) - s += 'USER: {0:} HOST: {1}\n'.format(user, host) + s += 'USER: {0:} HOST: {1}\n'.format(user, host) if numLibId is not None: s += 'NLIBID: {}\n'.format(numLibId) s += 'NPE_PIXEL_SATURATE: 100000\n' @@ -357,7 +360,7 @@ def simLibheader(self, numLibId=None, saturation_flag=1024, s += comments + '\n' s += 'BEGIN LIBGEN\n' return s - + def simLibFooter(self, numFields): """ """ @@ -367,14 +370,14 @@ def simLibFooter(self, numFields): def writeSimlib(self, filename, fields, doc='\n', comments='\n', fieldtype=None, mwebv=0., numLibId=None): - + num_fields = 0 with open(filename, 'w') as fh: # Write out the header to the simlib file simlib_header = self.simLibheader(numLibId=numLibId, doc=doc, comments=comments) fh.write(simlib_header) fh.flush() # RK - + # Now write the actual simlib data to file for field in fields: @@ -399,7 +402,7 @@ def writeSimlib(self, filename, fields, doc='\n', comments='\n', # fh.write(self.fieldfooter(fieldID)) num_fields += 1 - # Now write out the footer to the entire simlib file + # Now write out the footer to the entire simlib file simlib_footer = self.simLibFooter(num_fields) fh.write(simlib_footer) return num_fields @@ -417,7 +420,7 @@ def simlibs_for_fields(self, surveyPix, mwebv=0.): """Generator for simlib fields for a sequence of fields defined in a dataFrame called `surveyPix`. The dataFrame `surveyPix` must have the following columns `simlibId`, - `ra`, `dec` and must be sorted in increasing order of + `ra`, `dec` and must be sorted in increasing order of `simlibId`. Parameters @@ -469,7 +472,7 @@ def get_surveyPix(self, surveydf, numFields=15, rng=np.random.RandomState(0)): """ surveydf['simlibId'] = -1 len_surveydf = len(surveydf) - + if numFields <= len_surveydf : surveydf = surveydf.sample(n=numFields, replace=False, random_state=rng) @@ -482,19 +485,19 @@ def get_surveyPix(self, surveydf, numFields=15, rng=np.random.RandomState(0)): print('Printing original number of fields instead') sys.stdout.flush() - + hids = surveydf.reset_index()['hid'].values # xxxxxxxxxxx n_hids = len(hids) # xxxx .xyz hids_unique = list(set(hids)) n_unique = len(hids_unique) - + print(f"\n xxx len_surveydf={len_surveydf} n_hids={n_hids} n_unique={n_unique} " \ f"\n xxx hids = \n{hids}\n") sys.stdout.flush() # xxxxxxxxxxx - + surveydf.reset_index().set_index('hid') surveydf.loc[hids, 'simlibId'] = np.arange(len(hids)) return surveydf @@ -514,6 +517,7 @@ def randomSimlibs(self, numFields=50, fname='test.simlib', outfile=outfile, subset=self.subset, minVisits=minVisits, nside=256, mwebv=mwebv) + import pdb; pdb.set_trace() num_fields = self.writeSimlib(fname, fields, fieldtype=fieldtype, mwebv=mwebv) fields = self.sampleRegion(numFields=numFields, rng=rng, @@ -536,7 +540,7 @@ def __init__(self, simlibDict, simlibMetaData=None): @classmethod def fromSimlibFile(cls, simlibFileName): ''' - Constructor for class using an ASCII + Constructor for class using an ASCII Parameters ---------- @@ -555,7 +559,7 @@ def fromSimlibFile(cls, simlibFileName): file_header, file_data, file_footer = cls.read_simlibFile(simlibFileName) mydict = cls.getSimlibs(file_data) - meta = cls.simlibMetaData(file_header) + meta = cls.simlibMetaData(file_header) cls = cls(simlibDict=mydict, simlibMetaData=meta) cls.validate(file_footer) @@ -626,16 +630,16 @@ def getSimlibs(cls, file_data): s = FieldSimlib.fromSimlibString(strings) mydict[s.fieldID] = s - return mydict - + return mydict + @staticmethod def read_simlibFile(simlibfile): - + # slurp into a string with open(simlibfile) as f: ss = f.read() - + # split into header, footer and data fullfile = ss.split('BEGIN LIBGEN') file_header = fullfile[0] @@ -656,7 +660,7 @@ class FieldSimlib(object): """ Class to hold data corresponding to a particular fieldID (LIBID) of a SNANA SIMLIB file and methods. The fieldSimlib class for a particular field - has the following attributes, and may be instantiated by supplying these, + has the following attributes, and may be instantiated by supplying these, or can be conveniently constructed from the string corresponding to the description of this data in an SNANA simlib file using the constructor fromSimlibString: @@ -687,9 +691,9 @@ class FieldSimlib(object): def __init__(self, simlibdata, simlib_meta): """ Instantiate the class from the basic data - """ + """ - self.data = simlibdata + self.data = simlibdata self.meta = simlib_meta self.fieldID = self.meta['LIBID'] @@ -718,9 +722,9 @@ def fromSimlibString(cls, simlibstring): clsmeta = cls.libid_metadata(header_metadata) # Instantiate the class and make sure it works - myclass = cls(simlibdata=clsdata, simlib_meta=clsmeta) + myclass = cls(simlibdata=clsdata, simlib_meta=clsmeta) myclass.validate(footer) - return myclass + return myclass def validate(self, validate_string): """ @@ -734,7 +738,7 @@ def validate(self, validate_string): ---------- validate_string : string, mandatory footer obtained by splitting the simlib corresponding to the field - usually of the form + usually of the form """ val = eval(validate_string.split()[-1])