OiDBCuration.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3f7a0944-c87b-4dfa-8190-47bd391f29ee",
   "metadata": {},
   "source": [
    "## Update OiDB metadata using JMMC's CatalogAPI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b045df24-276b-4af5-9e0d-33ff7f2412b5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO - a2p2.jmmc.catalogs  - 2025-04-01 17:40:43,370 - catalogs.py:51 - Create catalog wrapper to access 'oidb' (production API at https://oidb.jmmc.fr/restxq/catalogs)\n"
     ]
    }
   ],
   "source": [
    "import pyvo\n",
    "from a2p2.jmmc import Catalog\n",
    "\n",
    "prod=False\n",
    "prod=True\n",
    "\n",
    "if prod==True:\n",
    "    oidbMainUrl=\"http://oidb.jmmc.fr\"\n",
    "    oidbTapUrl=\"http://tap.jmmc.fr/vollt/tap\"\n",
    "else:    \n",
    "    oidbMainUrl=\"http://oidb-beta.jmmc.fr\"\n",
    "    oidbTapUrl=\"http://tap-preprod.jmmc.fr/vollt/tap\"\n",
    "\n",
    "oidbTap = pyvo.dal.TAPService(oidbTapUrl)\n",
    "oidbCatalog = Catalog(\"oidb\", prod=prod)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "decde77a-e1f7-45f8-95a1-9325c0a163a3",
   "metadata": {},
   "source": [
    "### Define helper function to fix content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7372de60-6b57-4365-a18a-f3323bc64c5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fixGranule(granule, constraints=None):    \n",
    "    todo = {}\n",
    "    \n",
    "    # try by collection\n",
    "    obs_collection=granule[\"obs_collection\"]    \n",
    "    if obs_collection in collectionReference:\n",
    "        ref = collectionReference[obs_collection]        \n",
    "        for col in ref.keys():\n",
    "            ref_value     = ref[col]\n",
    "            granule_value = granule[col]\n",
    "            if ref_value != granule_value:\n",
    "                todo[col]=ref_value\n",
    "\n",
    "    # try by granule\n",
    "    granule_id=granule[\"id\"]    \n",
    "    if granule_id in granuleReferences:\n",
    "        print (f\"have to check id={granule_id}\")\n",
    "        ref = granuleReferences[granule_id]\n",
    "        for col in ref.keys():\n",
    "            ref_value     = ref[col]\n",
    "            granule_value = granule[col]\n",
    "            if ref_value != granule_value:\n",
    "                todo[col]=ref_value\n",
    "                \n",
    "    # try by todo if we a some constraints\n",
    "    if constraints:\n",
    "        for col in constraints.keys():\n",
    "            ref_value     = constraints[col]\n",
    "            granule_value = granule[col]\n",
    "            if ref_value != granule_value:\n",
    "                todo[col]=ref_value\n",
    "            \n",
    "    if todo:\n",
    "        todo[\"id\"]=str(granule[\"id\"])\n",
    "    return todo\n",
    "        \n",
    "def fixRecords(query, todo=None):\n",
    "    \"\"\" Walk accross granules selected by given query and return a list of dict so we can update the oidb catalog using api.updateRows()\"\"\"\n",
    "    updates=[]\n",
    "    granules = oidbTap.search(query).to_table()\n",
    "    print (f\"have to check {len(granules)} granule(s) for '{query}'\")\n",
    "\n",
    "    for granule in granules:\n",
    "        update = fixGranule(granule,todo)\n",
    "        if update:\n",
    "            updates.append(update)\n",
    "    if updates:\n",
    "        return updates\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8344283a-944a-482e-96fe-bab5870a70e6",
   "metadata": {},
   "source": [
    "### Fix some known cases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c25dae59-6a9a-4c86-a109-112f40dba618",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "have to check 776 granule(s) for 'SELECT ALL * FROM oidb AS t WHERE obs_collection='62aa7b9a-cd64-480e-aa7b-9acd64580eba''\n",
      "nothing to do for 'Update reference collections'\n",
      "have to check 0 granule(s) for 'SELECT ALL * FROM oidb AS t WHERE ( t.datapi LIKE '%Guillaume Mella%' )'\n",
      "nothing to do for 'Look for some records that may have bad metadata'\n",
      "have to check 0 granule(s) for 'SELECT ALL * FROM oidb AS t WHERE bib_reference='2023arXiv230308892C''\n",
      "nothing to do for 'Update by conditions references'\n"
     ]
    }
   ],
   "source": [
    "collectionReference={\n",
    "    #\"371c145f-7889-47d5-9c14-5f7889c7d510\":{\"datapi\":\"KRAUS\"},\n",
    "    #\"c5a5d133-a178-4949-a5d1-33a1781949e5\":{\"datapi\":\"SOULAIN\"},\n",
    "    #\"03a164ca-3110-446c-a164-ca3110746cdd\":{\"bib_reference\":\"2022Natur.602..403G\" },\n",
    "    #\"12803c68-6124-40ad-803c-68612470ad3d\":{\"bib_reference\":\"2022A&A...665A..32D\", \"calib_level\":3}\n",
    "    #\"76120870-3b88-4dfc-9208-703b881dfc57\":{\"datapi\":\"LAMBERTS\"}\n",
    "    #\"588ac90a-2e92-42f7-8ac9-0a2e92c2f701\": {\"datapi\":\"AUGEREAU\"},\n",
    "    #\"743b1acc-7463-464a-bb1a-cc7463664a0c\": {\"datapi\":\"AUGEREAU\"},\n",
    "    #\"a3e9ba2e-cd26-40f6-a9ba-2ecd2600f6b1\":{\"bib_reference\":\"2024A&A...681A..47V\", \"calib_level\":3}\n",
    "    #\"d259d56c-7304-4131-99d5-6c7304c13119\": {\"bib_reference\":\"2022MNRAS.510...82R\"}\n",
    "    \"62aa7b9a-cd64-480e-aa7b-9acd64580eba\": {\"datapi\":\"BAINES\"}\n",
    "}\n",
    "\n",
    "granuleReferences={\n",
    "    # 1713641:{ \"instrument_name\":\"MYSTIC\" }, # instrument name was not present in OIDB\n",
    "}\n",
    "byConditionsReferences={\n",
    "    #\"obs_collection='ddf733ce-758e-43c4-b733-ce758e33c4f3' AND em_min > 1.8e-6\" : { \"instrument_name\":\"MYSTIC\" },\n",
    "    #\"bib_reference='2024arXiv240507821P'\" : {\"bib_reference\":\"2024A&A...687A.306P\"}\n",
    "    \"bib_reference='2023arXiv230308892C'\" : {\"bib_reference\":\"2023A&A...675A..46C\"}\n",
    "}\n",
    "\n",
    "\n",
    "updates=[] # accumulates json update orders\n",
    "\n",
    "# Fix case 1 : \n",
    "reason=\"Update reference collections\"\n",
    "for collection_id in collectionReference.keys():\n",
    "    query = f\"SELECT ALL * FROM oidb AS t WHERE obs_collection='{collection_id}'\"\n",
    "    u=fixRecords(query)\n",
    "    if u:\n",
    "        updates+=u\n",
    "        print(f\"{len(u)} record(s) to update through catalogAPI for '{reason}'\")    \n",
    "    else:\n",
    "        print(f\"nothing to do for '{reason}'\")        \n",
    "\n",
    "# Fix case 2 : \n",
    "reason=\"Update by granule references\"\n",
    "for granule_id in granuleReferences.keys():\n",
    "    query = f\"SELECT ALL * FROM oidb AS t WHERE id='{granule_id}'\"\n",
    "    u=fixRecords(query)\n",
    "    if u:\n",
    "        updates+=u\n",
    "        print(f\"{len(u)} record(s) to update through catalogAPI for '{reason}'\")    \n",
    "    else:\n",
    "        print(f\"nothing to do for '{reason}'\")        \n",
    "        \n",
    "        \n",
    "# Fix case 3 : \n",
    "reason = \"Look for some records that may have bad metadata\"\n",
    "dataPiToFix=\"Guillaume Mella\" # this guy can't be a datapi, he often does hesitate between RA and DEC\n",
    "query = f\"SELECT ALL * FROM oidb AS t WHERE ( t.datapi LIKE '%{dataPiToFix}%' )\"\n",
    "u=fixRecords(query)\n",
    "if u:\n",
    "    updates+=u\n",
    "    print(f\"{len(u)} record(s) to update through catalogAPI for '{reason}'\")    \n",
    "else:\n",
    "    print(f\"nothing to do for '{reason}'\")   \n",
    "    \n",
    "\n",
    "# Fix case 4 : \n",
    "reason=\"Update by conditions references\"\n",
    "for condition in byConditionsReferences.keys():\n",
    "    query = f\"SELECT ALL * FROM oidb AS t WHERE {condition}\"\n",
    "    u=fixRecords(query, byConditionsReferences[condition])\n",
    "    if u:\n",
    "        updates+=u\n",
    "        print(f\"{len(u)} record(s) to update through catalogAPI for '{reason}', please check them and run next cell to apply changes.\")    \n",
    "    else:\n",
    "        print(f\"nothing to do for '{reason}'\")        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cb7b4137-734c-446d-ac91-587c335756e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "updates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "627756e1-8d95-4f6a-8a7f-130b8f44ceb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "if len(updates) > 0:\n",
    "    pass\n",
    "    # uncomment next line to apply changes    \n",
    "    #oidbCatalog.updateRows(updates)\n",
    "\n",
    "# That's all folks !!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7a8f31a-cb56-4f44-988f-2277d06fbaac",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80042843-3269-4049-ac84-701f015f19dd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df03c333-5b81-4d2a-9a93-0dbd46c66a24",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  },
  "vscode": {
   "interpreter": {
    "hash": "af71c39d2c5a77089dacebac0cd40852d8f53457e6ffaf39e53de42db39a43d2"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}