dr2cut_preproc.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "import os, glob\n",
    "import libstempo as t2\n",
    "\n",
    "import dr2lite_utils as dr2u"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook generates the `cut` and `freqfilt` datasets for the full IPTA.  It also splits out each dataset (including `classic`) by PTA.  This results in 12 datasets:\n",
    "\n",
    "`data/partim_${type}/${pta}`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# data preprocessing for `cut` analysis!\n",
    "\n",
    "start with full DR2, a.k.a. `classic`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DR2DATA = os.path.abspath('/home/pbaker/nanograv/data/DR2/')  # path to data local usage\n",
    "datadir = os.path.join(DR2DATA, 'release/VersionB')\n",
    "\n",
    "outdir = 'data/partim_classic'\n",
    "os.system('mkdir -p {}'.format(outdir));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### clean `.par`, combine `.tim`, make pulsar list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parfiles = glob.glob(datadir + '/J*/*IPTADR2.par')\n",
    "\n",
    "psr_names = []\n",
    "for p in parfiles:\n",
    "    name = p.split('/')[-2]\n",
    "    psr_names.append(name)\n",
    "    outfile = os.path.join(outdir, '{}.par'.format(name))\n",
    "    dr2u.clean_par(p, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "timfiles = glob.glob(datadir + '/J*/*IPTADR2.tim')\n",
    "\n",
    "for t in timfiles:\n",
    "    name = t.split('/')[-2]\n",
    "    outfile = os.path.join(outdir, '{}.tim'.format(name))\n",
    "    dr2u.combine_tim(t, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "psr_names.sort()\n",
    "\n",
    "psrfile = os.path.join(outdir, 'psrlist_classic.txt')\n",
    "with open(psrfile, 'w') as f:\n",
    "    for pname in psr_names:\n",
    "        f.write(\"{:s}\\n\".format(pname))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## generate `cut` dataset\n",
    "\n",
    "TODO: alter `dr2lite_utils.py` to allow for a list of flags to exclude (in addition to keep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "filt = {'group': \n",
    "                ['327_ASP', '430_ASP', 'L-wide_ASP', 'S-wide_ASP',\n",
    "                 '327_PUPPI', '430_PUPPI', 'L-wide_PUPPI',  'S-wide_PUPPI',\n",
    "                 'Rcvr_800_GASP', 'Rcvr1_2_GASP',\n",
    "                 'Rcvr_800_GUPPI', 'Rcvr1_2_GUPPI',\n",
    "                 'PDFB_10CM', 'PDFB_20CM', 'PDFB_40CM',\n",
    "                 'CPSR2_20CM', 'CPSR2_50CM',\n",
    "                 'WBCORR_10CM', 'WBCORR_20CM',\n",
    "                 'EFF.EBPP.1360', 'EFF.EBPP.1410', 'EFF.EBPP.2639',\n",
    "                 'JBO.DFB.1400', 'JBO.DFB.1520', 'JBO.DFB.5000',\n",
    "                 'NRT.BON.1400', 'NRT.BON.1600', 'NRT.BON.2000',\n",
    "                 'WSRT.P1.328', 'WSRT.P1.328.C', 'WSRT.P1.323.C',\n",
    "                 'WSRT.P1.382', 'WSRT.P1.382.C', 'WSRT.P1.367.C',\n",
    "                 'WSRT.P1.840', 'WSRT.P1.840.C',\n",
    "                 'WSRT.P1.1380', 'WSRT.P1.1380.C',\n",
    "                 'WSRT.P1.1380.1',\n",
    "                 'WSRT.P1.1380.2', 'WSRT.P1.1380.2.C',\n",
    "                 'WSRT.P1.2273.C',\n",
    "                ]\n",
    "        }  # list of all non-legacy backends (is this complete?)\n",
    "\n",
    "psrdict = {}\n",
    "for p in psr_names:\n",
    "    psrdict[p] = filt\n",
    "\n",
    "dr2u.make_dataset(psrdict, indir='data/partim_classic', outdir='data/partim_cut_IPTA', tmin=2, min_toas=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "generate pulsar list for `cut` dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datadir = 'data/partim_cut_IPTA'\n",
    "parfiles = glob.glob('data/partim_cut_IPTA/*.par')\n",
    "psrlist = []\n",
    "for p in parfiles:\n",
    "    name = p.split('/')[-1]\n",
    "    psrlist.append(name.split('.')[0])\n",
    "psrlist.sort()\n",
    "\n",
    "list_file = os.path.join(datadir, 'psrlist_cut_IPTA.txt')\n",
    "with open(list_file, 'w') as f:\n",
    "    for pname in psrlist:\n",
    "        f.write(\"{:s}\\n\".format(pname))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### split out each PTA from full `cut` dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datadir = 'data/partim_cut_IPTA'\n",
    "\n",
    "# get psrlist, if you skipped above steps\n",
    "list_file = os.path.join(datadir, 'psrlist_cut_IPTA.txt')\n",
    "with open(list_file, 'r') as f:\n",
    "    psrlist = [line.strip() for line in f]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PTAs = ['NANOGrav', 'PPTA', 'EPTA']\n",
    "outdir_base = 'data/partim_cut_{:s}'\n",
    "\n",
    "for pta in PTAs:\n",
    "    filt = {'pta':[pta]}\n",
    "    psrdict = {}\n",
    "    for p in psrlist:\n",
    "        psrdict[p] = filt\n",
    "    \n",
    "    # no additional filtering... just select by PTA\n",
    "    dr2u.make_dataset(psrdict, indir=datadir, outdir=outdir_base.format(pta),\n",
    "                      frequency_filter=False, min_toas=1, tmin=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outdir_base = 'data/partim_cut_{:s}'\n",
    "for pta in ['NANOGrav', 'PPTA', 'EPTA']:\n",
    "\n",
    "    datadir = outdir_base.format(pta)\n",
    "    parfiles = glob.glob('{:s}/*.par'.format(datadir))\n",
    "    psrnames = [p.split('/')[-1].split('.')[0] for p in parfiles]\n",
    "    psrnames.sort()\n",
    "\n",
    "    list_file = os.path.join(datadir, 'psrlist_cut_{:s}.txt'.format(pta))\n",
    "    with open(list_file, 'w') as f:\n",
    "        for pname in psrnames:\n",
    "            f.write(\"{:s}\\n\".format(pname))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## generate per-PTA `classic` datasets\n",
    "\n",
    "same per-PTA split out procedure as `cut`, but start with `classic`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datadir = 'data/partim_classic_IPTA'\n",
    "\n",
    "# get psrlist, if you skipped above steps\n",
    "list_file = os.path.join(datadir, 'psrlist_classic_IPTA.txt')\n",
    "with open(list_file, 'r') as f:\n",
    "    psrlist = [line.strip() for line in f]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "PTAs = ['NANOGrav', 'PPTA', 'EPTA']\n",
    "outdir_base = 'data/partim_classic_{:s}_test'\n",
    "\n",
    "for pta in PTAs:\n",
    "    filt = {'pta':[pta]}\n",
    "    psrdict = {}\n",
    "    for p in psrlist:\n",
    "        psrdict[p] = filt\n",
    "    \n",
    "    # no additional filtering... just select by PTA\n",
    "    dr2u.make_dataset(psrdict, indir=datadir, outdir=outdir_base.format(pta),\n",
    "                      frequency_filter=False, min_toas=2, tmin=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "generate pulsar lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outdir_base = 'data/partim_classic_{:s}'\n",
    "for pta in ['NANOGrav', 'PPTA', 'EPTA']:\n",
    "\n",
    "    datadir = outdir_base.format(pta)\n",
    "    parfiles = glob.glob('{:s}/*.par'.format(datadir))\n",
    "    psrnames = [p.split('/')[-1].split('.')[0] for p in parfiles]\n",
    "    psrnames.sort()\n",
    "\n",
    "    list_file = os.path.join(datadir, 'psrlist_classic_{:s}.txt'.format(pta))\n",
    "    with open(list_file, 'w') as f:\n",
    "        for pname in psrnames:\n",
    "            f.write(\"{:s}\\n\".format(pname))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## generate `freqfilt` dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datadir = 'data/partim_classic_IPTA'\n",
    "\n",
    "# get psrlist, if you skipped above steps\n",
    "list_file = os.path.join(datadir, 'psrlist_classic_IPTA.txt')\n",
    "with open(list_file, 'r') as f:\n",
    "    psrlist = [line.strip() for line in f]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "filt = {'pta':['NANOGrav', 'PPTA', 'EPTA']}  # keep all backends\n",
    "\n",
    "psrdict = {}\n",
    "for p in psrlist:\n",
    "    psrdict[p] = filt\n",
    "\n",
    "dr2u.make_dataset(psrdict, indir='data/partim_classic_IPTA', outdir='data/partim_freqfilt_IPTA',\n",
    "                  frequency_filter=True, tmin=2, min_toas=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datadir = 'data/partim_freqfilt_IPTA'\n",
    "parfiles = glob.glob('data/partim_freqfilt_IPTA/*.par')\n",
    "psrlist = []\n",
    "for p in parfiles:\n",
    "    name = p.split('/')[-1]\n",
    "    psrlist.append(name.split('.')[0])\n",
    "psrlist.sort()\n",
    "\n",
    "list_file = os.path.join(datadir, 'psrlist_freqfilt_IPTA.txt')\n",
    "with open(list_file, 'w') as f:\n",
    "    for pname in psrlist:\n",
    "        f.write(\"{:s}\\n\".format(pname))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### split out each PTA from full `freqfilt` dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "PTAs = ['NANOGrav', 'PPTA', 'EPTA']\n",
    "outdir_base = 'data/partim_classic_{:s}_test'\n",
    "\n",
    "for pta in PTAs:\n",
    "    filt = {'pta':[pta]}\n",
    "    psrdict = {}\n",
    "    for p in psrlist:\n",
    "        psrdict[p] = filt\n",
    "    \n",
    "    # no additional filtering... just select by PTA\n",
    "    dr2u.make_dataset(psrdict, indir=datadir, outdir=outdir_base.format(pta),\n",
    "                      frequency_filter=False, min_toas=2, tmin=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outdir_base = 'data/partim_freqfilt_{:s}'\n",
    "for pta in ['NANOGrav', 'PPTA', 'EPTA']:\n",
    "\n",
    "    datadir = outdir_base.format(pta)\n",
    "    parfiles = glob.glob('{:s}/*.par'.format(datadir))\n",
    "    psrnames = [p.split('/')[-1].split('.')[0] for p in parfiles]\n",
    "    psrnames.sort()\n",
    "\n",
    "    list_file = os.path.join(datadir, 'psrlist_freqfilt_{:s}.txt'.format(pta))\n",
    "    with open(list_file, 'w') as f:\n",
    "        for pname in psrnames:\n",
    "            f.write(\"{:s}\\n\".format(pname))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}