Text_Summarizer_Version_1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 953
        },
        "id": "v0ZQcs-80myZ",
        "outputId": "b61950a8-9263-4260-9e9a-f7b97a0d725b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.11/dist-packages (4.48.3)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.11/dist-packages (3.3.2)\n",
            "Requirement already satisfied: rouge_score in /usr/local/lib/python3.11/dist-packages (0.1.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.11/dist-packages (from transformers) (3.17.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.24.0 in /usr/local/lib/python3.11/dist-packages (from transformers) (0.28.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.11/dist-packages (from transformers) (1.26.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from transformers) (24.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.11/dist-packages (from transformers) (6.0.2)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.11/dist-packages (from transformers) (2024.11.6)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.11/dist-packages (from transformers) (2.32.3)\n",
            "Requirement already satisfied: tokenizers<0.22,>=0.21 in /usr/local/lib/python3.11/dist-packages (from transformers) (0.21.0)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.11/dist-packages (from transformers) (0.5.2)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.11/dist-packages (from transformers) (4.67.1)\n",
            "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.11/dist-packages (from datasets) (17.0.0)\n",
            "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.11/dist-packages (from datasets) (0.3.8)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (from datasets) (2.2.2)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.11/dist-packages (from datasets) (3.5.0)\n",
            "Requirement already satisfied: multiprocess<0.70.17 in /usr/local/lib/python3.11/dist-packages (from datasets) (0.70.16)\n",
            "Requirement already satisfied: fsspec<=2024.12.0,>=2023.1.0 in /usr/local/lib/python3.11/dist-packages (from fsspec[http]<=2024.12.0,>=2023.1.0->datasets) (2024.10.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.11/dist-packages (from datasets) (3.11.12)\n",
            "Requirement already satisfied: absl-py in /usr/local/lib/python3.11/dist-packages (from rouge_score) (1.4.0)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.11/dist-packages (from rouge_score) (3.9.1)\n",
            "Requirement already satisfied: six>=1.14.0 in /usr/local/lib/python3.11/dist-packages (from rouge_score) (1.17.0)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets) (2.4.6)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets) (1.3.2)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets) (25.1.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets) (1.5.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets) (6.1.0)\n",
            "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets) (0.2.1)\n",
            "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets) (1.18.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.11/dist-packages (from huggingface-hub<1.0,>=0.24.0->transformers) (4.12.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests->transformers) (3.4.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests->transformers) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests->transformers) (2.3.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests->transformers) (2025.1.31)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.11/dist-packages (from nltk->rouge_score) (8.1.8)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.11/dist-packages (from nltk->rouge_score) (1.4.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas->datasets) (2025.1)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas->datasets) (2025.1)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.11/dist-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
            "  warnings.warn(\n",
            "<ipython-input-4-770fc036378f>:48: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
            "  trainer = Trainer(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='359' max='359' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [359/359 52:17, Epoch 1/1]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>100</td>\n",
              "      <td>3.381400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>200</td>\n",
              "      <td>2.626200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>300</td>\n",
              "      <td>2.508100</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "!pip install transformers datasets rouge_score\n",
        "\n",
        "from datasets import load_dataset\n",
        "from transformers import T5Tokenizer, T5ForConditionalGeneration, Trainer, TrainingArguments\n",
        "from rouge_score import rouge_scorer\n",
        "import numpy as np\n",
        "\n",
        "# Load Subset of the Dataset\n",
        "try:\n",
        "    dataset = load_dataset(\"cnn_dailymail\", \"3.0.0\", split=\"train[:1%]\")\n",
        "except ValueError as e:\n",
        "    print(f\"Error loading dataset: {e}\")\n",
        "    print(\"Check dataset name, version, and Hugging Face Hub availability.\")\n",
        "    raise\n",
        "\n",
        "\n",
        "model_name = 't5-small'\n",
        "tokenizer = T5Tokenizer.from_pretrained(model_name)\n",
        "model = T5ForConditionalGeneration.from_pretrained(model_name)\n",
        "\n",
        "# Preprocess Data\n",
        "def preprocess_function(examples):\n",
        "    inputs = [\"summarize: \" + doc for doc in examples[\"article\"]]\n",
        "    model_inputs = tokenizer(inputs, max_length=256, truncation=True, padding=\"max_length\", return_tensors=\"pt\")\n",
        "\n",
        "    labels = tokenizer(examples[\"highlights\"], max_length=64, truncation=True, padding=\"max_length\", return_tensors=\"pt\")\n",
        "\n",
        "    model_inputs[\"labels\"] = labels[\"input_ids\"]\n",
        "    return model_inputs\n",
        "\n",
        "tokenized_datasets = dataset.map(preprocess_function, batched=True)\n",
        "\n",
        "# Training\n",
        "training_args = TrainingArguments(\n",
        "    output_dir=\"./results\",\n",
        "    evaluation_strategy=\"no\",\n",
        "    learning_rate=2e-5,\n",
        "    per_device_train_batch_size=8,\n",
        "    per_device_eval_batch_size=8,\n",
        "    num_train_epochs=1,\n",
        "    weight_decay=0.01,\n",
        "    save_steps=10000,\n",
        "    logging_steps=100,\n",
        "    push_to_hub=False,\n",
        "    report_to=\"none\"\n",
        ")\n",
        "\n",
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    train_dataset=tokenized_datasets,\n",
        "    tokenizer=tokenizer,\n",
        ")\n",
        "\n",
        "trainer.train()\n",
        "\n",
        "# Simplified Evaluation\n",
        "def compute_metrics(eval_pred):\n",
        "    predictions, labels = eval_pred\n",
        "    decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True)\n",
        "    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n",
        "    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n",
        "\n",
        "    scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True)\n",
        "    rouge_scores = []\n",
        "\n",
        "    for ref, pred in zip(decoded_labels, decoded_preds):\n",
        "        try:\n",
        "            score = scorer.score(ref, pred)\n",
        "            rouge_scores.append(score)\n",
        "        except Exception as e:\n",
        "            print(f\"Error calculating ROUGE score: {e}\")\n",
        "            print(f\"Reference: {ref}\")\n",
        "            print(f\"Prediction: {pred}\")\n",
        "            continue\n",
        "\n",
        "    # Calculate average ROUGE scores, handling cases where rouge_scores is empty\n",
        "    if rouge_scores:\n",
        "        avg_rouge1 = np.mean([score['rouge1'].fmeasure for score in rouge_scores])\n",
        "        avg_rouge2 = np.mean([score['rouge2'].fmeasure for score in rouge_scores])\n",
        "        avg_rougeL = np.mean([score['rougeL'].fmeasure for score in rouge_scores])\n",
        "\n",
        "        return {\n",
        "            'rouge1': avg_rouge1,\n",
        "            'rouge2': avg_rouge2,\n",
        "            'rougeL': avg_rougeL,\n",
        "        }\n",
        "    else:\n",
        "        return {\n",
        "            'rouge1': 0.0,\n",
        "            'rouge2': 0.0,\n",
        "            'rougeL': 0.0,\n",
        "            'error': \"No valid ROUGE scores calculated\"\n",
        "        }\n",
        "\n",
        "class CustomTrainer(Trainer):\n",
        "    def prediction_step(\n",
        "        self,\n",
        "        model,\n",
        "        inputs,\n",
        "        prediction_loss_only,\n",
        "        ignore_keys = None\n",
        "    ):\n",
        "        labels = inputs[\"labels\"].clone()\n",
        "        labels[labels == tokenizer.pad_token_id] = -100\n",
        "        inputs[\"labels\"] = labels\n",
        "        return super().prediction_step(\n",
        "            model, inputs, prediction_loss_only=prediction_loss_only, ignore_keys=ignore_keys\n",
        "        )\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "texts = [\"The U.S. economy added 850,000 jobs in June, a sign of continued recovery as businesses reopen and consumers spend more. The unemployment rate, however, ticked up slightly to 5.9% from 5.8% in May.\",\n",
        "        \"A massive wildfire in northern California has scorched over 150,000 acres, forcing thousands to evacuate. Firefighters are struggling to contain the blaze amid high temperatures and strong winds.\",\n",
        "        \"Scientists have discovered a new species of dinosaur in Argentina. The creature, named 'Llukalkan aliocranianus,' lived approximately 80 million years ago and is believed to have been a formidable predator.\",\n",
        "        \"The Tokyo 2020 Olympics, postponed due to the COVID-19 pandemic, are set to begin with strict health protocols in place. Athletes will undergo regular testing, and spectators will be limited to local residents.\",\n",
        "        \"A recent study suggests that drinking coffee may reduce the risk of developing Alzheimer's disease. Researchers found that participants who consumed higher amounts of caffeine had a lower incidence of the neurodegenerative condition.\",\n",
        "        \"The United Nations has called for an immediate ceasefire in the ongoing conflict in Yemen. The humanitarian crisis has worsened, with millions facing famine and limited access to medical supplies.\",\n",
        "        \"Tech giant Apple has announced plans to invest $1 billion in building a new campus in North Carolina. The facility is expected to create thousands of jobs and bolster the state's economy.\"]\n",
        "\n",
        "for text in texts:\n",
        "  input_ids = tokenizer.encode(\"summarize: \" + text, return_tensors=\"pt\").to(model.device)\n",
        "  output = model.generate(\n",
        "      input_ids,\n",
        "      max_length=100,\n",
        "      num_beams=4,\n",
        "      early_stopping=True,\n",
        "      do_sample=True,\n",
        "      temperature=0.9,\n",
        "      top_k=50,\n",
        "      top_p=0.95\n",
        "  )\n",
        "  summary = tokenizer.decode(output[0], skip_special_tokens=True)\n",
        "  print(summary)"
      ],
      "metadata": {
        "id": "1DTMUZOY1hs3",
        "outputId": "f16a4472-51bb-414c-bb99-b3ffea04c40c",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The U.S. economy added 850,000 jobs in June. The unemployment rate ticked slightly from 5.8% in May.\n",
            "blaze in northern California has scorched over 150,000 acres. Firefighters are struggling to contain fire amid high temperatures and strong winds.\n",
            "'Llukalkan aliocranianus' lived approximately 80 million years ago.\n",
            "Tokyo 2020 is postponed due to the COVID-19 pandemic.\n",
            "drinking coffee may reduce the risk of Alzheimer's.\n",
            "the conflict has worsened, with millions facing famine and limited access to medical supplies.\n",
            "Apple has announced plans to invest $1 billion in building a new campus in North Carolina. The facility will create thousands of jobs and bolster the state's economy.\n"
          ]
        }
      ]
    }
  ]
}