ic_fsc.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {},
      "outputs": [],
      "source": [
        "%load_ext autoreload\n",
        "%autoreload 2"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cJVqXw3dRSX-"
      },
      "source": [
        "# Configuring the Environment"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "KSKMp0o9RGyu"
      },
      "outputs": [],
      "source": [
        "!pip install --upgrade transformers datasets librosa "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BTOyqWt3RXfU"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import librosa\n",
        "from datasets import load_dataset, Dataset\n",
        "from transformers import Wav2Vec2ForSequenceClassification, HubertForSequenceClassification, Wav2Vec2FeatureExtractor\n",
        "\n",
        "from sklearn.metrics import accuracy_score\n",
        "import numpy as np \n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FTbn0utIRbgS"
      },
      "outputs": [],
      "source": [
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oJZvIVViRcOF"
      },
      "source": [
        "# Downloading & Unpacking the FSC Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "mneC_O6uRhSo"
      },
      "outputs": [],
      "source": [
        "!wget http://140.112.21.28:9000/fluent.tar.gz && tar -xf fluent.tar.gz\n",
        "!rm fluent.tar.gz"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "ZjfJ00D7Rjkw"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "\n",
        "fsc_folder = os.path.join('./', 'fluent_speech_commands_dataset')\n",
        "data_folder = f'{fsc_folder}/data/'\n",
        "speakers_folder = f\"{fsc_folder}/wavs/speakers\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sR9gO_wbRqG3"
      },
      "source": [
        "# Studying FSC Dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uCEGTzifR0EO"
      },
      "source": [
        "## Speaker Demographics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 177
        },
        "id": "1WRQTQZZRxFr",
        "outputId": "96a783e9-891c-4ba2-f892-70400986e1cf"
      },
      "outputs": [],
      "source": [
        "nRowsRead = None # specify 'None' if want to read whole file\n",
        "df_demographics = pd.read_csv(f'{data_folder}/speaker_demographics.csv', delimiter=',', nrows = nRowsRead)\n",
        "nRow, nCol = df_demographics.shape\n",
        "print(f'There are {nRow} rows and {nCol} columns')\n",
        "df_demographics.head(1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Wryu8FmzRzm0",
        "outputId": "43f46ab0-1b30-4e8f-eaaa-a357fad59ef2"
      },
      "outputs": [],
      "source": [
        "for col in df_demographics.columns:\n",
        "    if col!='speakerId':\n",
        "        print(f'Metadata: {col}')\n",
        "        frequency = dict(df_demographics[col].value_counts())\n",
        "        for k, v in frequency.items():\n",
        "              print(f'{k}: {v}')\n",
        "    print()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zott3YSoR8mc"
      },
      "source": [
        "## DataFrame for DivExplorer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 160
        },
        "id": "_lAaAxfQSAdd",
        "outputId": "d3eaefe7-45bc-42d3-8dee-07158762da6a"
      },
      "outputs": [],
      "source": [
        "nRowsRead = None # Specify 'None' if want to read whole file\n",
        "df_test = pd.read_csv(f'{data_folder}/test_data.csv', delimiter=',', nrows = nRowsRead, index_col = 0)\n",
        "df_test.dataframeName = 'test_data.csv'\n",
        "\n",
        "nRow, nCol = df_test.shape\n",
        "print(f'There are {nRow} rows and {nCol} columns')\n",
        "\n",
        "df_test['ID'] = df_test.index\n",
        "df_test.head(1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {},
      "outputs": [],
      "source": [
        "df_divExplorer = df_test.merge(df_demographics, on='speakerId')\n",
        "df_divExplorer['path'] = f'{fsc_folder}/' + df_divExplorer['path']\n",
        "\n",
        "df_divExplorer.set_index('ID', inplace=True)\n",
        "df_divExplorer = df_divExplorer.sort_values('ID')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "If-jTO6NSYGK"
      },
      "source": [
        "# Accuracy Evaluation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R2gPZZs-SaDr"
      },
      "source": [
        "## Inference"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "fLXrpEa3Sb5Z"
      },
      "outputs": [],
      "source": [
        "def map_to_array(example, audio_col = 'path'):\n",
        "    speech, _ = librosa.load(example[audio_col], sr=16000, mono=True)\n",
        "    example[\"speech\"] = speech\n",
        "    return example"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eCSRYY5nSemQ"
      },
      "outputs": [],
      "source": [
        "dataset = Dataset.from_pandas(df_divExplorer) \n",
        "dataset = dataset.map(lambda x: map_to_array(x, audio_col = 'path'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 52,
      "metadata": {
        "id": "5rLi-rOPSg6N"
      },
      "outputs": [],
      "source": [
        "target_model = 'wav2vec2-base'\n",
        "\n",
        "# Wav2Vec2 Large\n",
        "if target_model == 'wav2vec2-large':\n",
        "    model = Wav2Vec2ForSequenceClassification.from_pretrained(\"superb/wav2vec2-large-superb-ic\").cuda()\n",
        "    feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(\"superb/wav2vec2-large-superb-ic\")\n",
        "\n",
        "# Wav2Vec2 Base\n",
        "elif target_model == 'wav2vec2-base':\n",
        "    model = Wav2Vec2ForSequenceClassification.from_pretrained(\"superb/wav2vec2-base-superb-ic\").cuda()\n",
        "    feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(\"superb/wav2vec2-base-superb-ic\")\n",
        "\n",
        "# Hubert Large\n",
        "elif target_model == 'hubert-large':\n",
        "    model = HubertForSequenceClassification.from_pretrained(\"superb/hubert-large-superb-ic\").cuda()\n",
        "    feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(\"superb/hubert-large-superb-ic\")\n",
        "\n",
        "# Hubert Base\n",
        "elif target_model == 'hubert-base':\n",
        "    model = HubertForSequenceClassification.from_pretrained(\"superb/hubert-base-superb-ic\").cuda()\n",
        "    feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(\"superb/hubert-base-superb-ic\")\n",
        "else:\n",
        "    raise ValueError(f'{target_model} is not available')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 53,
      "metadata": {
        "id": "yrO--dGfSiz0"
      },
      "outputs": [],
      "source": [
        "def preprocess_function(examples):\n",
        "    inputs = feature_extractor(\n",
        "      examples,\n",
        "      sampling_rate=feature_extractor.sampling_rate, \n",
        "      padding=True, \n",
        "      return_tensors=\"pt\")\n",
        "      \n",
        "    return inputs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nmQkoBnuSkMM"
      },
      "outputs": [],
      "source": [
        "from tqdm import tqdm\n",
        "\n",
        "# compute attention masks and normalize the waveform if needed\n",
        "with torch.no_grad():\n",
        "    inputs = preprocess_function(dataset[0:48][\"speech\"]).to(device)\n",
        "    logits_concatenation = model(**inputs).logits\n",
        "    for i in tqdm(range(48, len(dataset), 48)):\n",
        "        inputs = preprocess_function(dataset[i:i+48][\"speech\"]).to(device)\n",
        "        logits = model(**inputs).logits\n",
        "        logits_concatenation = torch.cat((logits_concatenation, logits))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P_SSWxldSmc-"
      },
      "source": [
        "## Intent Accuracy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JjUcqGR-SpVq",
        "outputId": "50e6bdd6-cd77-48e4-8883-e57750d46f93"
      },
      "outputs": [],
      "source": [
        "action_ids = torch.argmax(logits_concatenation[:, :6], dim=-1).tolist()\n",
        "action_labels = [model.config.id2label[_id] for _id in action_ids]\n",
        "\n",
        "action_gt = list(df_divExplorer['action'].values)\n",
        "\n",
        "accuracy_score(action_gt, action_labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6T3Otk-TSqp0",
        "outputId": "79bc9ef2-56fa-4779-a79f-4b6fd8c2f840"
      },
      "outputs": [],
      "source": [
        "object_ids = torch.argmax(logits_concatenation[:, 6:20], dim=-1).tolist()\n",
        "object_labels = [model.config.id2label[_id + 6] for _id in object_ids]\n",
        "\n",
        "object_gt = list(df_divExplorer['object'].values)\n",
        "object_gt = [f'{x}_object' if x=='none' else x for x in object_gt]\n",
        "\n",
        "accuracy_score(object_gt, object_labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2os4Ue9iSr4L",
        "outputId": "e1454b83-6a71-4583-a177-02a1d16b32f5"
      },
      "outputs": [],
      "source": [
        "location_ids = torch.argmax(logits_concatenation[:, 20:24], dim=-1).tolist()\n",
        "location_labels = [model.config.id2label[_id + 20] for _id in location_ids]\n",
        "\n",
        "location_gt = list(df_divExplorer['location'].values)\n",
        "location_gt = [f'{x}_location' if x=='none' else x for x in location_gt]\n",
        "\n",
        "accuracy_score(location_gt, location_labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 58,
      "metadata": {
        "id": "8E7l-8ZYSzDN"
      },
      "outputs": [],
      "source": [
        "intents_predicted = [ action_labels[i]  + \" \" + object_labels[i] + \" \" + location_labels[i] for i in range(0, len(df_divExplorer))]\n",
        "\n",
        "intents_gt = [ action_gt[i]  + \" \" + object_gt[i] + \" \" + location_gt[i] for i in range(0, len(df_divExplorer))]\n",
        "\n",
        "is_correct = (np.array(intents_predicted) == np.array(intents_gt)).astype(int)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 59,
      "metadata": {
        "id": "1ZZPClq4S1XB"
      },
      "outputs": [],
      "source": [
        "df_divExplorer['prediction'] = is_correct"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 60,
      "metadata": {},
      "outputs": [],
      "source": [
        "output_folder = os.path.join('./data_precomputed', f'FSC_for_DivExplorer_{target_model}.csv')\n",
        "df_divExplorer.to_csv(output_folder, index=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x4esMiFKS6-z"
      },
      "source": [
        "# Extracting Signal Metadata"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 61,
      "metadata": {
        "id": "veB1EBG1S-TQ"
      },
      "outputs": [],
      "source": [
        "from signal_metadata_extraction import MetadataExtractor"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xrV4oQ9xTC5W"
      },
      "outputs": [],
      "source": [
        "metadata_extractor = MetadataExtractor()\n",
        "metadatas = []\n",
        "audio_col = 'path'\n",
        "\n",
        "from tqdm import tqdm\n",
        "for i in tqdm(range(len(df_divExplorer))):\n",
        "\n",
        "    audio_file = df_divExplorer[audio_col].iloc[i]\n",
        "    signal_metadata = metadata_extractor.signal_metadata(audio_file)\n",
        "    \n",
        "    sentence = df_divExplorer['transcription'].iloc[i] \n",
        "    text_metadata = metadata_extractor.text_metadata(sentence)\n",
        "    mixed_metadata = metadata_extractor.mixed_metadata(signal_metadata, text_metadata)\n",
        "    metadatas.append(signal_metadata + text_metadata + mixed_metadata)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LeYkc_vxTHds"
      },
      "outputs": [],
      "source": [
        "meta_cols = metadata_extractor.list_signal_metadata \\\n",
        "        + metadata_extractor.list_text_metadata \\\n",
        "        + metadata_extractor.list_sig_text_metadata\n",
        "df_divExplorer_metas = pd.concat([df_divExplorer, pd.DataFrame(metadatas, columns = meta_cols)], axis=1)\n",
        "\n",
        "print(\"Metadata columns: \", meta_cols)\n",
        "print(\"---\")\n",
        "print(\"Df for DivExplorer Columns: \", df_divExplorer_metas.columns)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z3yhus79TeKs"
      },
      "source": [
        "## Save Metadata in a CSV"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 64,
      "metadata": {
        "id": "APt1JCoCTQfD"
      },
      "outputs": [],
      "source": [
        "output_folder = os.path.join('./data_precomputed', f'FSC_for_DivExplorer_{target_model}')\n",
        "\n",
        "df_divExplorer_metas['path'] = df_divExplorer_metas['path'].str.replace(fsc_folder, \\\n",
        "    \"fluent_speech_commands_dataset\", regex = False)\n",
        "df_divExplorer_metas.to_csv(f'{output_folder}.csv', index=False)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "SUPERB - IC Task (FSC).ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3.10.4 ('amazon': conda)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.6"
    },
    "vscode": {
      "interpreter": {
        "hash": "a8dce71f01f4cf7d979b7741b7fb8d94cd1b30c77e0541871108952dcff484f0"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}