In [14]:
unflagged_LSTs()
Steven Murray & Josh Dillon, Last Revised May 2023
This notebook shows very simple summaries of the season in terms of observed antennas, times and frequencies, with flags. These are derived from the "scouting" flagger, which only looks at autocorrelations, full_day_auto_checker.ipynb in hera_notebook_templates. As such, it represents a minimal set of flags, but likely other failure modes that only appear in cross-correlations will be caught further down the pipeline. The main purpose of this investigation is to figure out which (subsets of which) JDs are worth analyzing.
Quick links to figures and tables:
import numpy as np
import pandas as pd
pd.set_option('display.max_rows', 1000)
import matplotlib.pyplot as plt
import matplotlib
import glob
import re
import os
from pathlib import Path
from astropy.time import Time
from IPython.display import display, HTML
from hera_notebook_templates.utils import status_colors, Antenna
import yaml
from astropy import units as u
from astropy.coordinates import EarthLocation
from pyuvdata import telescopes
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
display(HTML("<style>.container { width:100% !important; }</style>"))
The original YAML files were produced by "lightning scouting" with full_day_auto_checker.ipynb are in /lustre/aoc/projects/hera/h6c-analysis/day_flags/0/. However, manual inspection by JSD of all 177 notebooks (specifically Figures 2 and 3) resulted several changes, which are reflected hera_pipelines. The original flags are still in the YAMLs there, but commented out. Changes fell into the following categories:
RFI only briefly sends the metric for "too much broadband RFI" over threshold in the middle of the day, so it was decided to try to let full_day_rfi.ipynb try to handle it without a priori flags to avoid losing nearly half a day.
The whole day was near threshold, but just under, so we're manually flagging the whole day
JD flags were seemed more consistent with a DPSS filtering artifact than real broadband RFI (due to missing data or other flags), and so were scaled back or removed.
Flags seemed more consistent with a DPSS filtering artifact, but most of the array was dead anyway so we just flagged the whole day.
It may be that further modifications to a priori flags are necessary after running the full analysis pipeline and inspecting notebooks. We'll cross that bridge when we come to it.
# Where to find the a priori YAML files
yaml_folder = Path('/lustre/aoc/projects/hera/h6c-analysis/IDR2/src/hera_pipelines/pipelines/h6c/analysis/apriori_flags')
# load yamls in chronological order
yamls = sorted(yaml_folder.glob('*apriori_flags.yaml'))
print(f'Found {len(yamls)} a-priori YAMLS in {yaml_folder}')
Found 177 a-priori YAMLS in /lustre/aoc/projects/hera/h6c-analysis/IDR2/src/hera_pipelines/pipelines/h6c/analysis/apriori_flags
data = {}
for y in yamls:
int_jd = int(y.name[y.name.index("24"):y.name.index("24")+7])
with open(y, 'r') as fl:
data[int_jd] = yaml.load(fl, Loader=yaml.SafeLoader)
# Calculate the total amount of time that is flagged
hours_flagged = 0
hours_observed = 0
for jd, d in data.items():
hours_observed += (d['jd_range'][1] - d['jd_range'][0])*24
for flg in d['JD_flags']:
hours_flagged += (flg[1] - flg[0])*24
hours_flagged_frac = hours_flagged / (hours_observed)
print(f"Total number of hours observed: {hours_observed:.2f}")
print(f"Total number of hours flagged: {hours_flagged:.2f}")
print(f"Unflagged hours observed: {hours_observed - hours_flagged:.2f}")
print(f"Percentage of hours flagged: { hours_flagged_frac * 100:.2f}%")
Total number of hours observed: 1716.37 Total number of hours flagged: 400.86 Unflagged hours observed: 1315.50 Percentage of hours flagged: 23.36%
dt = 9.663677215576172 # seconds
lsts = np.arange(0, 24, dt / 60 / 60 * u.day.in_units(u.sday))
location = EarthLocation(*(telescopes.get_telescope('HERA').telescope_location * u.m))
lst_flags = {}
for jd, d in data.items():
jdmin, jdmax = d['jd_range']
tmin = Time(jdmin, format='jd', scale='utc', location=location)
tmax = Time(jdmax, format='jd', scale='utc', location=location)
lstmin = tmin.sidereal_time("apparent").hour
lstmax = tmax.sidereal_time("apparent").hour
if lstmax < lstmin:
lstmax += 24
thislst = np.where(lsts<lstmin, lsts + 24, lsts)
flags = thislst > lstmax
if np.all(flags):
print(jd, lstmin, lstmax)
for flgrange in d['JD_flags']:
tmin = Time(flgrange[0], format='jd', scale='utc', location=location)
tmax = Time(flgrange[1], format='jd', scale='utc', location=location)
lstmin = tmin.sidereal_time("apparent").hour
lstmax = tmax.sidereal_time("apparent").hour
if lstmax < lstmin:
lstmax += 24
flags[(thislst>=lstmin) & (thislst < lstmax)] = True
lst_flags[jd] = flags
all_days = np.array(list(range(min(data.keys()), max(data.keys()) + 1)))
lst_flag_array = np.zeros((len(all_days), len(lsts)), dtype=bool)
for i, day in enumerate(all_days):
lst_flag_array[i] = lst_flags.get(day, True)
def unflagged_LSTs():
plt.figure(figsize=(15, 5))
plt.imshow(~lst_flag_array, aspect='auto', interpolation='none', extent=(lsts.min(), lsts.max(), min(all_days) - 2459000, max(all_days) - 2459000), cmap='binary_r', origin='lower')
plt.xlabel("LST (hour)")
plt.ylabel("Night (JD - 2459000)")
plt.title("Unflagged JDs and LSTs Over the Season (White)")
This figure shows the range of LSTs that are observed and not a priori flagged on each day over the season. Flags in the nominally observed ranges of time are due to missing data, the sun being up, or excess temporal variability (likely attributable to lightning). Additional times might later be flagged in the full pipeline due to more sporadic broadband RFI.
unflagged_LSTs()
def LST_coverage():
plt.figure(figsize=(15, 5))
plt.plot(lsts, np.sum(~lst_flag_array, axis=0))
plt.ylim([0, 1.05 * np.max(np.sum(~lst_flag_array, axis=0))])
plt.xlim([0, 24])
plt.grid()
plt.xlabel("LST (hour)")
plt.ylabel('Number of Unflagged Nights')
This figure shows how often each LST was observed after flagging (subsets of) nights. This represents the maximum number of different nights that could be binned together at a given LST, though additional times may be flagged in the full-pipeline due to, e.g., broadband RFI.
LST_coverage()
These only include antennas that are flagged because of the autocorrelations and flagged for their entire nights.
ngood_ants = {'Jnn': {}, 'Jee': {}}
ntot_ants = {'Jnn': {}, 'Jee': {}}
for pol in ['Jnn', "Jee"]:
for jd, d in data.items():
ntot_ants[pol][jd] = len([a for a in d['all_ant'] if a[1] ==pol])
ngood_ants[pol][jd] = ntot_ants[pol][jd] - len([a for a in d['ex_ants'] if a[1] == pol])
def ant_flag_plot():
plt.figure(figsize=(15, 5))
for i, pol in enumerate(ngood_ants):
if i==1:
width=-0.4
else:
width=0.4
plt.bar(all_days-2459000, [ngood_ants[pol].get(d, 0) for d in all_days], align='edge', width=width, label=f'Good {pol}')
plt.bar(all_days-2459000, [ntot_ants[pol].get(d, 0) - ngood_ants[pol].get(d, 0) for d in all_days],
bottom=[ngood_ants[pol].get(d, 0) for d in all_days], align='edge', width=width, label=f'Bad {pol}')
plt.xlabel("JD - 2459000")
plt.ylabel("Number of Antennas")
plt.legend()
This plot shows the number of antennas for each polarization that are observed and flagged/unflagged on each night for each polarization. Note that antenna flagging was done only on autocorrelations, so it is likely an undercount because it does not include all flags due to effects that might only show up in the cross-correlations, like node timing issues, packet loss, and non-redundancy.
ant_flag_plot()
Times are flagged because of excess broadband structure (e.g. lightning) and because the sun was up.
ngood_ant_hours = {'Jnn': {}, 'Jee': {}}
ntot_ant_hours = {'Jnn': {}, 'Jee': {}}
for pol in ['Jnn', "Jee"]:
for jd, d in data.items():
ntot_ant_hours[pol][jd] = ntot_ants[pol][jd] * (d['jd_range'][1] - d['jd_range'][0])*24
ngood_ant_hours[pol][jd] = ngood_ants[pol][jd] * np.max([0, ((d['jd_range'][1] - d['jd_range'][0])*24 - sum((x[1]- x[0])*24 for x in d['JD_flags']))])
def ant_hour_plot():
plt.figure(figsize=(15, 5))
for i, pol in enumerate(ngood_ants):
if i==1:
width=-0.4
else:
width=0.4
plt.bar(all_days-2459000, [ngood_ant_hours[pol].get(d, 0) for d in all_days], align='edge', width=width, label=f'Good {pol}')
plt.bar(all_days-2459000, [ntot_ant_hours[pol].get(d, 0) - ngood_ant_hours[pol].get(d, 0) for d in all_days], bottom=[ngood_ant_hours[pol].get(d, 0) for d in all_days], align='edge', width=width, label=f'Bad {pol}')
plt.xlabel("JD - 2459000")
plt.ylabel("Number of Antenna-Hours")
plt.legend()
This plot shows the number of antenna-hours for each polarization that are observed and flagged/unflagged on each night for each polarization. Note that antenna flagging was done only on autocorrelations, so it is likely an undercount because it does not include all flags due to effects that might only show up in the cross-correlations, like node timing issues, packet loss, and non-redundancy. Time-flagging includes sun-up data and excess temporal variability (likely due to lightning). Additional times might later be flagged in the full pipeline due to more sporadic broadband RFI.
ant_hour_plot()
print(f'Average antenna-hours per observed night (ee): {np.mean(list(ngood_ant_hours["Jee"].values())):.2f}')
print(f'Average antenna-hours per observed night (nn): {np.mean(list(ngood_ant_hours["Jnn"].values())):.2f}')
print(f'Total antenna-hours (ee): {np.sum(list(ngood_ant_hours["Jee"].values())):.2f}')
print(f'Total antenna-hours (nn): {np.sum(list(ngood_ant_hours["Jnn"].values())):.2f}')
Average antenna-hours per observed night (ee): 1062.57 Average antenna-hours per observed night (nn): 1030.03 Total antenna-hours (ee): 188074.95 Total antenna-hours (nn): 182314.75
Channels flags are based on narrowband RFI detected in individual 2-integration files. Only channels flagged for an entire night are included here. 87.5 -- 108.0 MHz was flagged a priori due to FM radio contamination.
ngood_ant_hour_chans = {'Jnn': {}, 'Jee': {}}
ntot_ant_hour_chans = {'Jnn': {}, 'Jee': {}}
for pol in ['Jnn', "Jee"]:
for jd, d in data.items():
ntot_ant_hour_chans[pol][jd] = ntot_ant_hours[pol][jd] * 1500
ngood_ant_hour_chans[pol][jd] = ngood_ant_hours[pol][jd] * 1500*((d['freq_range'][1] - d['freq_range'][0]) - sum((x[1]- x[0]) for x in d['freq_flags']))/(d['freq_range'][1] - d['freq_range'][0])
def ant_hour_chan_plot():
plt.figure(figsize=(15, 5))
for i, pol in enumerate(ngood_ants):
if i==1:
width=-0.4
else:
width=0.4
plt.bar(all_days-2459000, [ngood_ant_hour_chans[pol].get(d, 0) for d in all_days], align='edge', width=width, label=f"Good {pol}")
plt.bar(all_days-2459000, [ntot_ant_hour_chans[pol].get(d, 0) - ngood_ant_hour_chans[pol].get(d, 0) for d in all_days], bottom=[ngood_ant_hour_chans[pol].get(d, 0) for d in all_days], align='edge', width=width, label=f"Bad {pol}")
plt.xlabel("JD - 2459000")
plt.ylabel("Number of Antenna-Hour-Channels")
plt.legend()
This plot shows the number of antenna-hour-channels for each polarization that are observed and flagged/unflagged on each night for each polarization. Note that antenna flagging was done only on autocorrelations, so it is likely an undercount because it does not include all flags due to effects that might only show up in the cross-correlations, like node timing issues, packet loss, and non-redundancy. Time-flagging includes sun-up data and excess temporal variability (likely due to lightning). Additional times might later be flagged in the full pipeline due to more sporadic broadband RFI. Frequency flagging only includes channels that are 100% flagged over the night in quesiton. This includes a handful of channels with persistent emitters, as well as the range from 87.5--108 MHz, the FM band.
ant_hour_chan_plot()
print(f"Fraction of total possible observed time unflagged (ee): {np.sum(list(ngood_ant_hour_chans['Jee'].values())) / np.sum(list(ntot_ant_hour_chans['Jee'].values())):.2%}")
print(f"Fraction of total possible observed time unflagged (nn): {np.sum(list(ngood_ant_hour_chans['Jnn'].values())) / np.sum(list(ntot_ant_hour_chans['Jnn'].values())):.2%}")
Fraction of total possible observed time unflagged (ee): 49.44% Fraction of total possible observed time unflagged (nn): 47.92%
fmin, fmax = next(iter(data.values()))['freq_range']
freqs = np.linspace(fmin, fmax, 1500)
freq_flag_array = np.zeros((len(all_days), len(freqs)), dtype=bool)
for i, day in enumerate(all_days):
if day not in data:
freq_flag_array[i] = True
else:
for flgs in data[day]['freq_flags']:
freq_flag_array[i][(freqs >= flgs[0]) & (freqs < flgs[1])] = True
def channel_flag_plot():
plt.figure(figsize=(15, 5))
plt.imshow(~freq_flag_array, aspect='auto', interpolation='none', origin='lower', cmap='binary_r',
extent=(freqs.min()/1e6, freqs.max()/1e6, min(all_days)-2459000, max(all_days)-2459000))
plt.xlabel("Frequency (MHz)")
plt.ylabel("JD - 2459000")
plt.title("Completely Flagged Channels Over Nights");
This plot shows which channels are flagged in every file on any given night--channels that are only sometimes (or even often) flagged might become fully-flagged in later processing when we attempt to make a self-consistent flagging mask for a whole night. The most prominent feature here is the FM band, which is flagged a priori from 87.5--108 MHz due to the difficulty of identifying low-level RFI amongst extremely bright RFI.
channel_flag_plot()
These parameters help us make the decision about whether a night is worth analyzing. If the amount of unflagged data is too small, it is easier to simply give up on the day and move on.
min_unflagged_hours = 4
min_unflagged_antpols_frac = .4
min_unflagged_antpolhours_frac = .3
jd_col = list(range(min(all_days), max(all_days) + 1))
hours_obs = []
unflagged_hours = []
unflagged_time_frac = []
antpols_obs = []
unflagged_antpols = []
unflagged_antpols_frac = []
unflagged_antpol_hours = []
unflagged_antpolhours_frac = []
night_flagged = []
sum_unflagged_nights = 0
sum_unflagged_hours = 0
sum_unflagged_antpol_hours = 0
sum_unflagged_antpolsq_hours = 0
for JD in jd_col:
if JD in data:
hrs_obs = (data[JD]['jd_range'][1] - data[JD]['jd_range'][0]) * 24
hrs_flagged = np.sum([np.diff(f) for f in data[JD]['JD_flags']]) * 24
hours_obs.append(f"{hrs_obs:.2f}")
unflagged_hours.append(f"{np.max([hrs_obs - hrs_flagged, 0]):.2f}")
unflagged_time_frac.append(f"{np.max([hrs_obs - hrs_flagged, 0]) / hrs_obs:.1%}")
ap_obs = len(data[JD]['all_ant'])
ap_flagged = len(data[JD]['ex_ants'])
antpols_obs.append(f"{ap_obs}")
unflagged_antpols.append(f"{ap_obs - ap_flagged}")
unflagged_antpols_frac.append(f"{(ap_obs - ap_flagged) / ap_obs:.1%}")
unflagged_antpol_hours.append(f"{np.max([hrs_obs - hrs_flagged, 0]) * (ap_obs - ap_flagged):.1f}")
unflagged_antpolhours_frac.append(f"{np.max([hrs_obs - hrs_flagged, 0]) * (ap_obs - ap_flagged) / hrs_obs / ap_obs:.1%}")
is_flagged = (hrs_obs - hrs_flagged) < min_unflagged_hours
is_flagged |= ((ap_obs - ap_flagged) / ap_obs) < min_unflagged_antpols_frac
is_flagged |= ((hrs_obs - hrs_flagged) * (ap_obs - ap_flagged) / hrs_obs / ap_obs) < min_unflagged_antpolhours_frac
night_flagged.append(is_flagged)
sum_unflagged_nights += float(~is_flagged)
sum_unflagged_hours += float(~is_flagged) * np.max([hrs_obs - hrs_flagged, 0])
sum_unflagged_antpol_hours += float(~is_flagged) * np.max([hrs_obs - hrs_flagged, 0]) * (ap_obs - ap_flagged)
sum_unflagged_antpolsq_hours += float(~is_flagged) * np.max([hrs_obs - hrs_flagged, 0]) * (ap_obs - ap_flagged)**2
else:
for arr in [hours_obs, unflagged_hours, unflagged_time_frac, antpols_obs, unflagged_antpols,
unflagged_antpols_frac, unflagged_antpol_hours, unflagged_antpolhours_frac]:
arr.append('-')
night_flagged.append(True)
df = pd.DataFrame({'JD': jd_col,
'Hours Observed': hours_obs,
'Unflagged Hours': unflagged_hours,
'Unflagged Time %': unflagged_time_frac,
'Antpols in Data': antpols_obs,
'Unflagged Antpols': unflagged_antpols,
'Unflagged Antpol %': unflagged_antpols_frac,
'Unflagged Antpol-Hours': unflagged_antpol_hours,
'Unflagged Antpol-Time %': unflagged_antpolhours_frac,
})
# background_gradient(cmap='viridis', vmax=mean_round_modz_cut * 3, vmin=0, axis=None, subset=z_score_cols) \
utp_vals = [float(entry[:-1]) if entry != '-' else 0 for entry in df['Unflagged Time %']]
uap_vals = [float(entry[:-1]) if entry != '-' else 0 for entry in df['Unflagged Antpol %']]
uatp_vals = [float(entry[:-1]) if entry != '-' else 0 for entry in df['Unflagged Antpol-Time %']]
table = df.style.hide().background_gradient(subset=['Unflagged Time %'], cmap='RdYlGn', gmap=utp_vals) \
.background_gradient(subset=['Unflagged Antpol %'], cmap='RdYlGn', gmap=uap_vals) \
.background_gradient(subset=['Unflagged Antpol-Time %'], cmap='RdYlGn', gmap=uatp_vals) \
.background_gradient(subset=['JD'], cmap='RdYlGn_r', gmap=night_flagged) \
.set_table_styles([dict(selector="th",props=[('max-width', f'60pt')])])
This table summarizes the per night statistics for the amount of data, unflagged data, and flagging fractions along various axes. Note that the level of flagging is likely an underestimate, since this initial data investigation was done using only autocorrelations and it might be missing some flags that the full pipeline would find. Nights with dashes instead of data were not observed. JDs in green are likely worth analyzing; JDs in red are likely not.
HTML(table.to_html())
| JD | Hours Observed | Unflagged Hours | Unflagged Time % | Antpols in Data | Unflagged Antpols | Unflagged Antpol % | Unflagged Antpol-Hours | Unflagged Antpol-Time % |
|---|---|---|---|---|---|---|---|---|
| 2459847 | 9.99 | 9.71 | 97.1% | 360 | 276 | 76.7% | 2679.0 | 74.5% |
| 2459848 | 9.27 | 7.50 | 80.9% | 360 | 277 | 76.9% | 2077.3 | 62.2% |
| 2459849 | 10.55 | 8.44 | 80.0% | 354 | 265 | 74.9% | 2235.7 | 59.9% |
| 2459850 | - | - | - | - | - | - | - | - |
| 2459851 | 9.99 | 8.82 | 88.3% | 312 | 180 | 57.7% | 1587.7 | 50.9% |
| 2459852 | 11.93 | 0.00 | 0.0% | 360 | 283 | 78.6% | 0.0 | 0.0% |
| 2459853 | 9.65 | 9.65 | 100.0% | 360 | 259 | 71.9% | 2499.6 | 71.9% |
| 2459854 | 5.99 | 5.47 | 91.2% | 360 | 271 | 75.3% | 1481.1 | 68.6% |
| 2459855 | 7.11 | 6.78 | 95.4% | 360 | 254 | 70.6% | 1722.9 | 67.3% |
| 2459856 | 9.50 | 4.48 | 47.1% | 360 | 277 | 76.9% | 1240.6 | 36.3% |
| 2459857 | 9.00 | 9.00 | 100.0% | 360 | 35 | 9.7% | 314.8 | 9.7% |
| 2459858 | 9.99 | 9.99 | 100.0% | 360 | 275 | 76.4% | 2748.3 | 76.4% |
| 2459859 | 9.99 | 9.99 | 100.0% | 360 | 273 | 75.8% | 2728.3 | 75.8% |
| 2459860 | 9.99 | 9.99 | 100.0% | 360 | 265 | 73.6% | 2648.4 | 73.6% |
| 2459861 | 9.99 | 9.99 | 100.0% | 360 | 258 | 71.7% | 2578.4 | 71.7% |
| 2459862 | 8.00 | 7.57 | 94.7% | 360 | 262 | 72.8% | 1983.3 | 68.9% |
| 2459863 | 9.99 | 7.12 | 71.2% | 360 | 277 | 76.9% | 1971.2 | 54.8% |
| 2459864 | 9.99 | 9.99 | 100.0% | 360 | 263 | 73.1% | 2627.7 | 73.0% |
| 2459865 | 9.99 | 0.00 | 0.0% | 360 | 257 | 71.4% | 0.0 | 0.0% |
| 2459866 | 9.99 | 9.95 | 99.5% | 360 | 265 | 73.6% | 2635.6 | 73.3% |
| 2459867 | 9.99 | 9.93 | 99.3% | 360 | 269 | 74.7% | 2670.3 | 74.2% |
| 2459868 | 9.99 | 9.91 | 99.2% | 360 | 251 | 69.7% | 2487.6 | 69.1% |
| 2459869 | 9.99 | 6.90 | 69.0% | 360 | 244 | 67.8% | 1682.7 | 46.8% |
| 2459870 | 9.99 | 9.87 | 98.8% | 360 | 255 | 70.8% | 2516.9 | 70.0% |
| 2459871 | 9.99 | 9.85 | 98.6% | 360 | 267 | 74.2% | 2631.1 | 73.1% |
| 2459872 | 9.69 | 8.72 | 90.0% | 360 | 269 | 74.7% | 2345.4 | 67.3% |
| 2459873 | 9.99 | 9.83 | 98.3% | 366 | 268 | 73.2% | 2633.7 | 72.0% |
| 2459874 | 9.99 | 9.81 | 98.2% | 366 | 262 | 71.6% | 2570.6 | 70.3% |
| 2459875 | 9.99 | 0.00 | 0.0% | 366 | 275 | 75.1% | 0.0 | 0.0% |
| 2459876 | 9.99 | 9.53 | 95.4% | 366 | 257 | 70.2% | 2450.5 | 67.0% |
| 2459877 | - | - | - | - | - | - | - | - |
| 2459878 | 9.39 | 9.10 | 96.9% | 402 | 289 | 71.9% | 2629.1 | 69.7% |
| 2459879 | 9.06 | 8.86 | 97.7% | 402 | 245 | 60.9% | 2169.6 | 59.6% |
| 2459880 | 9.48 | 9.27 | 97.7% | 402 | 284 | 70.6% | 2631.7 | 69.0% |
| 2459881 | 11.93 | 9.69 | 81.2% | 402 | 274 | 68.2% | 2655.2 | 55.4% |
| 2459882 | 9.93 | 9.68 | 97.5% | 402 | 274 | 68.2% | 2651.5 | 66.4% |
| 2459883 | 9.93 | 9.66 | 97.3% | 402 | 264 | 65.7% | 2551.2 | 63.9% |
| 2459884 | 9.93 | 9.65 | 97.2% | 402 | 263 | 65.4% | 2538.0 | 63.6% |
| 2459885 | 9.93 | 0.00 | 0.0% | 402 | 294 | 73.1% | 0.0 | 0.0% |
| 2459886 | 9.93 | 5.07 | 51.0% | 402 | 309 | 76.9% | 1566.0 | 39.2% |
| 2459887 | 9.93 | 9.25 | 93.1% | 402 | 296 | 73.6% | 2738.1 | 68.6% |
| 2459888 | 9.92 | 0.00 | 0.0% | 402 | 299 | 74.4% | 0.0 | 0.0% |
| 2459889 | 9.92 | 9.58 | 96.5% | 402 | 298 | 74.1% | 2855.0 | 71.6% |
| 2459890 | 9.92 | 9.57 | 96.4% | 402 | 296 | 73.6% | 2831.8 | 71.0% |
| 2459891 | 9.93 | 9.56 | 96.3% | 402 | 288 | 71.6% | 2753.0 | 69.0% |
| 2459892 | 9.93 | 5.02 | 50.6% | 402 | 284 | 70.6% | 1426.4 | 35.7% |
| 2459893 | 9.93 | 9.54 | 96.1% | 402 | 279 | 69.4% | 2661.0 | 66.7% |
| 2459894 | 9.93 | 9.93 | 100.0% | 402 | 293 | 72.9% | 2909.3 | 72.9% |
| 2459895 | 9.93 | 0.00 | 0.0% | 402 | 289 | 71.9% | 0.0 | 0.0% |
| 2459896 | 9.92 | 9.50 | 95.7% | 402 | 291 | 72.4% | 2764.5 | 69.3% |
| 2459897 | 9.92 | 9.49 | 95.6% | 402 | 286 | 71.1% | 2714.7 | 68.0% |
| 2459898 | 9.92 | 9.48 | 95.6% | 402 | 298 | 74.1% | 2824.6 | 70.8% |
| 2459899 | - | - | - | - | - | - | - | - |
| 2459900 | 9.93 | 0.00 | 0.0% | 402 | 298 | 74.1% | 0.0 | 0.0% |
| 2459901 | - | - | - | - | - | - | - | - |
| 2459902 | 9.92 | 9.45 | 95.2% | 402 | 296 | 73.6% | 2797.7 | 70.1% |
| 2459903 | 9.93 | 9.45 | 95.1% | 402 | 294 | 73.1% | 2777.2 | 69.6% |
| 2459904 | 9.93 | 9.44 | 95.1% | 402 | 289 | 71.9% | 2729.2 | 68.4% |
| 2459905 | 9.93 | 9.44 | 95.1% | 402 | 289 | 71.9% | 2727.6 | 68.3% |
| 2459906 | 9.93 | 9.43 | 95.0% | 402 | 299 | 74.4% | 2820.4 | 70.7% |
| 2459907 | 9.93 | 9.43 | 94.9% | 402 | 303 | 75.4% | 2857.3 | 71.5% |
| 2459908 | 9.93 | 9.43 | 94.9% | 402 | 283 | 70.4% | 2668.0 | 66.8% |
| 2459909 | 9.93 | 9.42 | 94.9% | 402 | 305 | 75.9% | 2872.9 | 72.0% |
| 2459910 | 9.93 | 9.93 | 100.0% | 402 | 303 | 75.4% | 3008.6 | 75.4% |
| 2459911 | 9.93 | 0.00 | 0.0% | 402 | 306 | 76.1% | 0.0 | 0.0% |
| 2459912 | 9.93 | 9.41 | 94.8% | 402 | 297 | 73.9% | 2794.4 | 70.0% |
| 2459913 | 9.93 | 5.86 | 59.0% | 400 | 306 | 76.5% | 1792.3 | 45.1% |
| 2459914 | 9.93 | 0.00 | 0.0% | 400 | 312 | 78.0% | 0.0 | 0.0% |
| 2459915 | 9.93 | 6.45 | 64.9% | 400 | 310 | 77.5% | 1998.8 | 50.3% |
| 2459916 | 9.93 | 9.41 | 94.7% | 400 | 314 | 78.5% | 2953.5 | 74.4% |
| 2459917 | 5.92 | 5.92 | 100.0% | 400 | 306 | 76.5% | 1811.2 | 76.5% |
| 2459918 | 9.93 | 0.00 | 0.0% | 400 | 295 | 73.8% | 0.0 | 0.0% |
| 2459919 | 9.11 | 7.74 | 85.0% | 402 | 278 | 69.2% | 2152.5 | 58.8% |
| 2459920 | 8.98 | 8.98 | 100.0% | 402 | 245 | 60.9% | 2199.9 | 60.9% |
| 2459921 | 9.93 | 9.66 | 97.2% | 402 | 173 | 43.0% | 1670.4 | 41.8% |
| 2459922 | 9.93 | 8.57 | 86.3% | 402 | 275 | 68.4% | 2357.1 | 59.0% |
| 2459923 | 9.93 | 6.49 | 65.3% | 402 | 310 | 77.1% | 2010.5 | 50.4% |
| 2459924 | 9.93 | 4.96 | 49.9% | 402 | 314 | 78.1% | 1556.8 | 39.0% |
| 2459925 | 9.93 | 0.00 | 0.0% | 402 | 313 | 77.9% | 0.0 | 0.0% |
| 2459926 | 9.93 | 0.00 | 0.0% | 402 | 314 | 78.1% | 0.0 | 0.0% |
| 2459927 | 0.80 | 0.80 | 100.0% | 402 | 289 | 71.9% | 230.4 | 71.9% |
| 2459928 | 9.93 | 9.57 | 96.4% | 402 | 307 | 76.4% | 2938.7 | 73.6% |
| 2459929 | 9.81 | 0.00 | 0.0% | 402 | 311 | 77.4% | 0.0 | 0.0% |
| 2459930 | 9.93 | 9.55 | 96.2% | 402 | 241 | 60.0% | 2301.8 | 57.7% |
| 2459931 | 9.92 | 9.54 | 96.1% | 402 | 232 | 57.7% | 2212.7 | 55.5% |
| 2459932 | 2.85 | 2.85 | 100.0% | 402 | 234 | 58.2% | 666.5 | 58.2% |
| 2459933 | 9.05 | 8.65 | 95.6% | 402 | 242 | 60.2% | 2093.7 | 57.5% |
| 2459934 | 9.92 | 9.51 | 95.9% | 402 | 241 | 60.0% | 2292.7 | 57.5% |
| 2459935 | 9.92 | 9.50 | 95.8% | 402 | 310 | 77.1% | 2945.8 | 73.9% |
| 2459936 | 8.65 | 0.00 | 0.0% | 402 | 293 | 72.9% | 0.0 | 0.0% |
| 2459937 | 9.93 | 0.00 | 0.0% | 402 | 293 | 72.9% | 0.0 | 0.0% |
| 2459938 | 9.92 | 9.48 | 95.5% | 402 | 295 | 73.4% | 2796.9 | 70.1% |
| 2459939 | 9.92 | 9.47 | 95.5% | 402 | 293 | 72.9% | 2775.6 | 69.6% |
| 2459940 | 9.93 | 9.35 | 94.2% | 402 | 278 | 69.2% | 2600.7 | 65.1% |
| 2459941 | 9.93 | 9.93 | 100.0% | 402 | 301 | 74.9% | 2988.8 | 74.9% |
| 2459942 | 9.93 | 6.15 | 61.9% | 402 | 298 | 74.1% | 1831.9 | 45.9% |
| 2459943 | 9.93 | 9.45 | 95.1% | 402 | 298 | 74.1% | 2815.8 | 70.5% |
| 2459944 | 9.93 | 9.45 | 95.1% | 402 | 292 | 72.6% | 2758.3 | 69.1% |
| 2459945 | 9.93 | 9.44 | 95.1% | 402 | 297 | 73.9% | 2803.9 | 70.2% |
| 2459946 | 9.93 | 9.44 | 95.0% | 402 | 295 | 73.4% | 2783.5 | 69.7% |
| 2459947 | 9.92 | 4.98 | 50.2% | 402 | 296 | 73.6% | 1474.7 | 37.0% |
| 2459948 | 9.92 | 0.00 | 0.0% | 402 | 295 | 73.4% | 0.0 | 0.0% |
| 2459949 | 9.00 | 9.00 | 100.0% | 392 | 295 | 75.3% | 2653.6 | 75.3% |
| 2459950 | 9.93 | 9.42 | 94.9% | 392 | 297 | 75.8% | 2799.2 | 71.9% |
| 2459951 | 9.92 | 9.42 | 94.9% | 392 | 289 | 73.7% | 2723.0 | 70.0% |
| 2459952 | 9.92 | 9.42 | 94.9% | 392 | 296 | 75.5% | 2788.1 | 71.7% |
| 2459953 | 9.93 | 9.42 | 94.8% | 392 | 285 | 72.7% | 2685.3 | 69.0% |
| 2459954 | 9.93 | 9.42 | 94.8% | 392 | 300 | 76.5% | 2826.6 | 72.6% |
| 2459955 | 9.93 | 9.42 | 94.8% | 392 | 299 | 76.3% | 2816.4 | 72.3% |
| 2459956 | 9.93 | 0.00 | 0.0% | 392 | 299 | 76.3% | 0.0 | 0.0% |
| 2459957 | 9.93 | 9.42 | 94.9% | 392 | 300 | 76.5% | 2827.4 | 72.6% |
| 2459958 | 9.93 | 9.42 | 94.9% | 392 | 292 | 74.5% | 2752.0 | 70.7% |
| 2459959 | 9.93 | 9.43 | 94.9% | 392 | 182 | 46.4% | 1715.8 | 44.1% |
| 2459960 | 9.92 | 7.34 | 73.9% | 392 | 287 | 73.2% | 2105.5 | 54.1% |
| 2459961 | 9.93 | 0.00 | 0.0% | 392 | 278 | 70.9% | 0.0 | 0.0% |
| 2459962 | 9.92 | 0.00 | 0.0% | 392 | 288 | 73.5% | 0.0 | 0.0% |
| 2459963 | 9.92 | 0.00 | 0.0% | 392 | 293 | 74.7% | 0.0 | 0.0% |
| 2459964 | 9.92 | 0.00 | 0.0% | 392 | 278 | 70.9% | 0.0 | 0.0% |
| 2459965 | 9.93 | 9.18 | 92.4% | 392 | 304 | 77.6% | 2789.2 | 71.7% |
| 2459966 | 9.92 | 6.49 | 65.4% | 392 | 303 | 77.3% | 1965.1 | 50.5% |
| 2459967 | 9.92 | 9.46 | 95.3% | 392 | 298 | 76.0% | 2819.0 | 72.5% |
| 2459968 | 9.92 | 0.00 | 0.0% | 392 | 290 | 74.0% | 0.0 | 0.0% |
| 2459969 | 9.93 | 9.48 | 95.4% | 392 | 307 | 78.3% | 2909.1 | 74.7% |
| 2459970 | 9.93 | 9.48 | 95.5% | 392 | 306 | 78.1% | 2902.1 | 74.5% |
| 2459971 | 9.92 | 9.49 | 95.6% | 392 | 298 | 76.0% | 2827.8 | 72.7% |
| 2459972 | 9.92 | 9.49 | 95.7% | 392 | 299 | 76.3% | 2838.9 | 73.0% |
| 2459973 | 9.93 | 9.51 | 95.7% | 392 | 298 | 76.0% | 2833.4 | 72.8% |
| 2459974 | 9.93 | 9.52 | 95.8% | 392 | 303 | 77.3% | 2883.4 | 74.1% |
| 2459975 | 9.92 | 9.38 | 94.5% | 392 | 303 | 77.3% | 2841.9 | 73.1% |
| 2459976 | 9.93 | 8.82 | 88.9% | 392 | 298 | 76.0% | 2629.4 | 67.6% |
| 2459977 | 9.92 | 7.81 | 78.7% | 392 | 301 | 76.8% | 2350.0 | 60.4% |
| 2459978 | 9.93 | 9.56 | 96.2% | 392 | 303 | 77.3% | 2895.6 | 74.4% |
| 2459979 | 9.92 | 9.57 | 96.4% | 392 | 305 | 77.8% | 2917.9 | 75.0% |
| 2459980 | 9.93 | 5.50 | 55.4% | 396 | 295 | 74.5% | 1623.4 | 41.3% |
| 2459981 | 9.93 | 9.59 | 96.5% | 396 | 305 | 77.0% | 2923.7 | 74.3% |
| 2459982 | 9.93 | 0.00 | 0.0% | 396 | 299 | 75.5% | 0.0 | 0.0% |
| 2459983 | 9.93 | 5.25 | 52.8% | 396 | 305 | 77.0% | 1600.6 | 40.7% |
| 2459984 | 9.92 | 8.38 | 84.4% | 396 | 301 | 76.0% | 2520.9 | 64.1% |
| 2459985 | 9.93 | 9.64 | 97.1% | 396 | 308 | 77.8% | 2968.1 | 75.5% |
| 2459986 | 9.94 | 6.95 | 69.9% | 396 | 310 | 78.3% | 2155.3 | 54.8% |
| 2459987 | 9.93 | 9.51 | 95.7% | 396 | 299 | 75.5% | 2842.1 | 72.2% |
| 2459988 | 9.93 | 9.68 | 97.5% | 396 | 299 | 75.5% | 2894.3 | 73.6% |
| 2459989 | 9.93 | 9.69 | 97.6% | 396 | 303 | 76.5% | 2937.0 | 74.7% |
| 2459990 | 9.94 | 9.71 | 97.7% | 396 | 299 | 75.5% | 2903.9 | 73.8% |
| 2459991 | 9.93 | 9.65 | 97.2% | 396 | 292 | 73.7% | 2818.9 | 71.7% |
| 2459992 | - | - | - | - | - | - | - | - |
| 2459993 | - | - | - | - | - | - | - | - |
| 2459994 | 9.94 | 9.77 | 98.3% | 396 | 287 | 72.5% | 2805.1 | 71.3% |
| 2459995 | 9.93 | 9.79 | 98.6% | 396 | 271 | 68.4% | 2653.0 | 67.5% |
| 2459996 | 9.93 | 9.81 | 98.8% | 396 | 265 | 66.9% | 2598.6 | 66.1% |
| 2459997 | 9.93 | 9.82 | 98.9% | 396 | 287 | 72.5% | 2818.9 | 71.7% |
| 2459998 | 9.92 | 9.84 | 99.1% | 396 | 314 | 79.3% | 3088.3 | 78.6% |
| 2459999 | 6.42 | 0.00 | 0.0% | 396 | 131 | 33.1% | 0.0 | 0.0% |
| 2460000 | - | - | - | - | - | - | - | - |
| 2460001 | - | - | - | - | - | - | - | - |
| 2460002 | - | - | - | - | - | - | - | - |
| 2460003 | - | - | - | - | - | - | - | - |
| 2460004 | - | - | - | - | - | - | - | - |
| 2460005 | - | - | - | - | - | - | - | - |
| 2460006 | - | - | - | - | - | - | - | - |
| 2460007 | 9.94 | 7.96 | 80.1% | 396 | 199 | 50.3% | 1583.9 | 40.2% |
| 2460008 | 9.93 | 0.00 | 0.0% | 396 | 186 | 47.0% | 0.0 | 0.0% |
| 2460009 | 9.93 | 9.93 | 100.0% | 396 | 192 | 48.5% | 1906.5 | 48.5% |
| 2460010 | 9.93 | 9.93 | 100.0% | 396 | 188 | 47.5% | 1867.7 | 47.5% |
| 2460011 | 9.94 | 9.94 | 100.0% | 396 | 198 | 50.0% | 1968.2 | 50.0% |
| 2460012 | 9.94 | 9.94 | 100.0% | 396 | 268 | 67.7% | 2664.0 | 67.7% |
| 2460013 | 9.93 | 9.93 | 100.0% | 396 | 297 | 75.0% | 2950.6 | 75.0% |
| 2460014 | 4.59 | 4.59 | 100.0% | 396 | 297 | 75.0% | 1364.1 | 75.0% |
| 2460015 | 9.93 | 9.93 | 100.0% | 396 | 295 | 74.5% | 2930.8 | 74.5% |
| 2460016 | 9.93 | 9.93 | 100.0% | 396 | 298 | 75.3% | 2960.6 | 75.3% |
| 2460017 | 9.93 | 9.93 | 100.0% | 396 | 288 | 72.7% | 2861.2 | 72.7% |
| 2460018 | 9.93 | 7.33 | 73.8% | 396 | 280 | 70.7% | 2052.7 | 52.2% |
| 2460019 | 9.94 | 9.94 | 100.0% | 396 | 288 | 72.7% | 2862.8 | 72.7% |
| 2460020 | 9.93 | 9.93 | 100.0% | 396 | 287 | 72.5% | 2851.3 | 72.5% |
| 2460021 | 9.93 | 5.22 | 52.5% | 396 | 291 | 73.5% | 1517.8 | 38.6% |
| 2460022 | - | - | - | - | - | - | - | - |
| 2460023 | 9.93 | 9.93 | 100.0% | 396 | 218 | 55.1% | 2165.8 | 55.1% |
| 2460024 | 9.93 | 9.93 | 100.0% | 396 | 280 | 70.7% | 2780.2 | 70.7% |
| 2460025 | 9.93 | 9.93 | 100.0% | 396 | 285 | 72.0% | 2831.4 | 72.0% |
| 2460026 | 9.93 | 0.00 | 0.0% | 396 | 294 | 74.2% | 0.0 | 0.0% |
| 2460027 | 9.93 | 0.00 | 0.0% | 398 | 294 | 73.9% | 0.0 | 0.0% |
| 2460028 | 9.93 | 5.18 | 52.1% | 398 | 294 | 73.9% | 1521.6 | 38.5% |
| 2460029 | - | - | - | - | - | - | - | - |
| 2460030 | 9.93 | 9.93 | 100.0% | 398 | 291 | 73.1% | 2891.0 | 73.1% |
| 2460031 | 9.93 | 9.93 | 100.0% | 398 | 287 | 72.1% | 2851.3 | 72.1% |
| 2460032 | 9.94 | 9.94 | 100.0% | 398 | 297 | 74.6% | 2952.2 | 74.6% |
| 2460033 | 9.93 | 9.93 | 100.0% | 398 | 305 | 76.6% | 3030.1 | 76.6% |
| 2460034 | 9.93 | 9.93 | 100.0% | 396 | 301 | 76.0% | 2990.4 | 76.0% |
| 2460035 | 9.93 | 9.93 | 100.0% | 396 | 302 | 76.3% | 3000.3 | 76.3% |
| 2460036 | 9.94 | 9.94 | 100.0% | 396 | 298 | 75.3% | 2962.2 | 75.3% |
| 2460037 | 9.93 | 8.36 | 84.1% | 396 | 295 | 74.5% | 2465.9 | 62.7% |
| 2460038 | 9.93 | 9.93 | 100.0% | 396 | 289 | 73.0% | 2869.6 | 73.0% |
print(f'• H6C has (up to) {sum_unflagged_nights:.0f} nights worth analyzing out of {len(yamls)} nights of data in the season.')
print(f'• H6C has (up to) a combined {sum_unflagged_hours:.1f} hours of data worth analyzing.')
print(f'• Those nights have (up to) an average of {sum_unflagged_antpol_hours / sum_unflagged_hours:.1f} unflagged antpols.')
print(f'• H6C has (up to) a combined {sum_unflagged_antpol_hours:.2e} antpol-hours of data worth analyzing.')
print(f'• H6C has (up to) a combined {sum_unflagged_antpolsq_hours:.2e} antpol^2-hours of data worth analyzing.')
print(f'• The nights worth analyzing are:\n{[jd for jd, nf in zip(jd_col, night_flagged) if not nf]}')
• H6C has (up to) 147 nights worth analyzing out of 177 nights of data in the season. • H6C has (up to) a combined 1312.2 hours of data worth analyzing. • Those nights have (up to) an average of 281.3 unflagged antpols. • H6C has (up to) a combined 3.69e+05 antpol-hours of data worth analyzing. • H6C has (up to) a combined 1.05e+08 antpol^2-hours of data worth analyzing. • The nights worth analyzing are: [2459847, 2459848, 2459849, 2459851, 2459853, 2459854, 2459855, 2459856, 2459858, 2459859, 2459860, 2459861, 2459862, 2459863, 2459864, 2459866, 2459867, 2459868, 2459869, 2459870, 2459871, 2459872, 2459873, 2459874, 2459876, 2459878, 2459879, 2459880, 2459881, 2459882, 2459883, 2459884, 2459886, 2459887, 2459889, 2459890, 2459891, 2459892, 2459893, 2459894, 2459896, 2459897, 2459898, 2459902, 2459903, 2459904, 2459905, 2459906, 2459907, 2459908, 2459909, 2459910, 2459912, 2459913, 2459915, 2459916, 2459917, 2459919, 2459920, 2459921, 2459922, 2459923, 2459924, 2459928, 2459930, 2459931, 2459933, 2459934, 2459935, 2459938, 2459939, 2459940, 2459941, 2459942, 2459943, 2459944, 2459945, 2459946, 2459947, 2459949, 2459950, 2459951, 2459952, 2459953, 2459954, 2459955, 2459957, 2459958, 2459959, 2459960, 2459965, 2459966, 2459967, 2459969, 2459970, 2459971, 2459972, 2459973, 2459974, 2459975, 2459976, 2459977, 2459978, 2459979, 2459980, 2459981, 2459983, 2459984, 2459985, 2459986, 2459987, 2459988, 2459989, 2459990, 2459991, 2459994, 2459995, 2459996, 2459997, 2459998, 2460007, 2460009, 2460010, 2460011, 2460012, 2460013, 2460014, 2460015, 2460016, 2460017, 2460018, 2460019, 2460020, 2460021, 2460023, 2460024, 2460025, 2460028, 2460030, 2460031, 2460032, 2460033, 2460034, 2460035, 2460036, 2460037, 2460038]