Mini_Muse.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "gradient": {
          "editing": false,
          "id": "ac5a4cf0-d9d2-47b5-9633-b53f8d99a4d2",
          "kernelId": ""
        },
        "id": "SiTIpPjArIyr"
      },
      "source": [
        "# Mini Muse (ver. 1.0)\n",
        "\n",
        "***\n",
        "\n",
        "Powered by tegridy-tools: https://github.com/asigalov61/tegridy-tools\n",
        "\n",
        "***\n",
        "\n",
        "Credit for GPT2-RGA code used in this colab goes out @ Sashmark97 https://github.com/Sashmark97/midigen and @ Damon Gwinn https://github.com/gwinndr/MusicTransformer-Pytorch\n",
        "\n",
        "***\n",
        "\n",
        "WARNING: This complete implementation is a functioning model of the Artificial Intelligence. Please excercise great humility, care, and respect. https://www.nscai.gov/\n",
        "\n",
        "***\n",
        "\n",
        "#### Project Los Angeles\n",
        "\n",
        "#### Tegridy Code 2022\n",
        "\n",
        "***"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "gradient": {
          "editing": false,
          "id": "fa0a611c-1803-42ae-bdf6-a49b5a4e781b",
          "kernelId": ""
        },
        "id": "gOd93yV0sGd2"
      },
      "source": [
        "# (Setup Environment)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "gradient": {
          "editing": false,
          "id": "39411b40-9e39-416e-8fe4-d40f733e7956",
          "kernelId": ""
        },
        "id": "lw-4aqV3sKQG"
      },
      "outputs": [],
      "source": [
        "#@title nvidia-smi gpu check\n",
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "gradient": {
          "editing": false,
          "id": "a1a45a91-d909-4fd4-b67a-5e16b971d179",
          "kernelId": ""
        },
        "id": "fX12Yquyuihc",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Install all dependencies (run only once per session)\n",
        "\n",
        "!git clone https://github.com/asigalov61/Mini-Muse\n",
        "\n",
        "!pip install torch\n",
        "\n",
        "!pip install tqdm\n",
        "!pip install matplotlib\n",
        "!pip install torch-summary\n",
        "\n",
        "!apt install fluidsynth #Pip does not work for some reason. Only apt works\n",
        "!pip install midi2audio\n",
        "!pip install pretty_midi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "gradient": {
          "editing": false,
          "id": "b8207b76-9514-4c07-95db-95a4742e52c5",
          "kernelId": ""
        },
        "id": "z7n9vnKmug1J"
      },
      "outputs": [],
      "source": [
        "#@title Import all needed modules\n",
        "\n",
        "print('Loading needed modules. Please wait...')\n",
        "import os\n",
        "import random\n",
        "import copy\n",
        "\n",
        "from collections import OrderedDict\n",
        "\n",
        "from tqdm import tqdm\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from torchsummary import summary\n",
        "\n",
        "print('Loading TMIDIX module...')\n",
        "os.chdir('/content/Mini-Muse')\n",
        "\n",
        "import TMIDIX\n",
        "from GPT2RGAX import *\n",
        "\n",
        "from midi2audio import FluidSynth\n",
        "import pretty_midi\n",
        "import librosa.display\n",
        "from IPython.display import Audio\n",
        "\n",
        "os.chdir('/content/')"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (UNZIP)"
      ],
      "metadata": {
        "id": "iJ4xaSoA9A9s"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Unzip pre-trained Mini Muse Model\n",
        "%cd /content/Mini-Muse/Model\n",
        "\n",
        "print('=' * 70)\n",
        "print('Unzipping pre-trained model...Please wait...')\n",
        "print('=' * 70)\n",
        "\n",
        "!cat /content/Mini-Muse/Model/Mini-Muse-Trained-Model.zip* > Mini-Muse-Trained-Model.zip\n",
        "print('=' * 70)\n",
        "\n",
        "!unzip -j Mini-Muse-Trained-Model.zip\n",
        "print('=' * 70)\n",
        "\n",
        "print('Done! Enjoy! :)')\n",
        "print('=' * 70)\n",
        "%cd /content/"
      ],
      "metadata": {
        "cellView": "form",
        "id": "hvybRqy05RnI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Unzip Mini Muse Training Data\n",
        "%cd /content/Mini-Muse/Training-Data\n",
        "\n",
        "print('=' * 70)\n",
        "print('Unzipping pre-trained model...Please wait...')\n",
        "print('=' * 70)\n",
        "\n",
        "!cat /content/Mini-Muse/Training-Data/Mini-Muse-Training-Data.zip* > Mini-Muse-Training-Data.zip\n",
        "print('=' * 70)\n",
        "\n",
        "!unzip -j Mini-Muse-Training-Data.zip\n",
        "print('=' * 70)\n",
        "\n",
        "print('Done! Enjoy! :)')\n",
        "print('=' * 70)\n",
        "%cd /content/"
      ],
      "metadata": {
        "cellView": "form",
        "id": "vwkHqgL38Dyu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mdKFoeke9L7H"
      },
      "source": [
        "# (LOAD)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Load/Reload the model\n",
        "\n",
        "full_path_to_model_checkpoint = \"/content/Mini-Muse/Model/Mini_Muse_Trained_Model_88000_steps_0.6129_loss.pth\" #@param {type:\"string\"}\n",
        "\n",
        "print('Loading the model...')\n",
        "config = GPTConfig(512, \n",
        "                   1024,\n",
        "                   dim_feedforward=1024,\n",
        "                   n_layer=16, \n",
        "                   n_head=16, \n",
        "                   n_embd=1024,\n",
        "                   enable_rpr=True,\n",
        "                   er_len=1024)\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "model = GPT(config)\n",
        "\n",
        "state_dict = torch.load(full_path_to_model_checkpoint, map_location=device)\n",
        "\n",
        "new_state_dict = OrderedDict()\n",
        "for k, v in state_dict.items():\n",
        "    name = k[7:] #remove 'module'\n",
        "    new_state_dict[name] = v\n",
        "\n",
        "model.load_state_dict(new_state_dict)\n",
        "\n",
        "model.to(device)\n",
        "\n",
        "model.eval()\n",
        "\n",
        "print('Done!')\n",
        "\n",
        "summary(model)"
      ],
      "metadata": {
        "cellView": "form",
        "id": "-NLe35B0b9a5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Load/Reload training data\n",
        "print('Loading training data...')\n",
        "train_data = TMIDIX.Tegridy_Any_Pickle_File_Reader('/content/Mini-Muse/Training-Data/Mini-Muse-Training-Data')\n",
        "print('Done!')"
      ],
      "metadata": {
        "cellView": "form",
        "id": "Rxj4-7hN93x0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# (GENERATE)"
      ],
      "metadata": {
        "id": "RCKJSbl4erb0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Generate Prime\n",
        "number_of_prime_tokens = 256 #@param {type:\"slider\", min:16, max:512, step:16}\n",
        "\n",
        "#@markdown Select desired combination of instruments (less is better)\n",
        "\n",
        "Piano = True #@param {type:\"boolean\"}\n",
        "Guitar = False #@param {type:\"boolean\"}\n",
        "Bass = False #@param {type:\"boolean\"}\n",
        "Violin = False #@param {type:\"boolean\"}\n",
        "Cello = False #@param {type:\"boolean\"}\n",
        "Harp = False #@param {type:\"boolean\"}\n",
        "Trumpet = False #@param {type:\"boolean\"}\n",
        "Clarinet = False #@param {type:\"boolean\"}\n",
        "Flute = False #@param {type:\"boolean\"}\n",
        "Drums = False #@param {type:\"boolean\"}\n",
        "Choir = False #@param {type:\"boolean\"}\n",
        "\n",
        "instruments = []\n",
        "\n",
        "if Piano:\n",
        "  instruments += [0]\n",
        "\n",
        "if Guitar:\n",
        "  instruments += [1]\n",
        "\n",
        "if Bass:\n",
        "  instruments += [2]\n",
        "\n",
        "if Violin:\n",
        "  instruments += [3]\n",
        "\n",
        "if Cello:\n",
        "  instruments += [4]\n",
        "\n",
        "if Harp:\n",
        "  instruments += [5]\n",
        "\n",
        "if Trumpet:\n",
        "  instruments += [6]\n",
        "\n",
        "if Clarinet:\n",
        "  instruments += [7]\n",
        "\n",
        "if Flute:\n",
        "  instruments += [8]\n",
        "\n",
        "if Drums:\n",
        "  instruments += [9]\n",
        "\n",
        "if Choir:\n",
        "  instruments += [10]\n",
        "\n",
        "iidx = []\n",
        "\n",
        "print('Looking for matching primes...')\n",
        "\n",
        "for i in tqdm(range(0, len(train_data), 1025)):\n",
        "  instr = sorted(list(set([y // 10 for y in train_data[i+1:i+1025:4][:number_of_prime_tokens]])))\n",
        "  if instr == sorted(instruments):\n",
        "    iidx.append(i)\n",
        "\n",
        "print('Found', len(iidx), 'matching primes...')\n",
        "\n",
        "iindex = random.choice(iidx)\n",
        "print('Selected prime #', iindex // 1025)\n",
        "print('Prime index:', iindex)\n",
        "\n",
        "out1 = train_data[iindex:iindex+number_of_prime_tokens+1]\n",
        "\n",
        "if len(out1) != 0:\n",
        "    \n",
        "    song = out1\n",
        "    song_f = []\n",
        "    time = 0\n",
        "    dur = 0\n",
        "    vel = 0\n",
        "    pitch = 0\n",
        "    channel = 0\n",
        "\n",
        "    son = []\n",
        "\n",
        "    song1 = []\n",
        "\n",
        "    for s in song:\n",
        "      if s > 127:\n",
        "        son.append(s)\n",
        "\n",
        "      else:\n",
        "        if len(son) == 4:\n",
        "          song1.append(son)\n",
        "        son = []\n",
        "        son.append(s)\n",
        "    \n",
        "    for s in song1:\n",
        "\n",
        "        channel = s[0] // 10\n",
        "\n",
        "        vel = (s[0] % 10) * 16\n",
        "\n",
        "        time += (s[1]-128) * 16\n",
        "            \n",
        "        dur = (s[2] - 256) * 32\n",
        "        \n",
        "        pitch = (s[3] - 384)\n",
        "                                  \n",
        "        song_f.append(['note', time, dur, channel, pitch, vel ])\n",
        "\n",
        "    detailed_stats = TMIDIX.Tegridy_SONG_to_MIDI_Converter(song_f,\n",
        "                                                        output_signature = 'Mini Muse',  \n",
        "                                                        output_file_name = '/content/Mini-Muse-Music-Composition', \n",
        "                                                        track_name='Project Los Angeles',\n",
        "                                                        list_of_MIDI_patches=[0, 24, 32, 40, 42, 46, 56, 71, 73, 0, 53, 0, 0, 0, 0, 0],\n",
        "                                                        number_of_ticks_per_quarter=500)\n",
        "\n",
        "    print('Done!')\n",
        "\n",
        "print('Displaying resulting composition...')\n",
        "fname = '/content/Mini-Muse-Music-Composition'\n",
        "\n",
        "pm = pretty_midi.PrettyMIDI(fname + '.mid')\n",
        "\n",
        "# Retrieve piano roll of the MIDI file\n",
        "piano_roll = pm.get_piano_roll()\n",
        "\n",
        "plt.figure(figsize=(14, 5))\n",
        "librosa.display.specshow(piano_roll, x_axis='time', y_axis='cqt_note', fmin=1, hop_length=160, sr=16000, cmap=plt.cm.hot)\n",
        "plt.title(fname)\n",
        "\n",
        "FluidSynth(\"/usr/share/sounds/sf2/FluidR3_GM.sf2\", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))\n",
        "Audio(str(fname + '.wav'), rate=16000)"
      ],
      "metadata": {
        "cellView": "form",
        "id": "9xENgPG2SlWq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Single Continuation Block Generator\n",
        "\n",
        "#@markdown NOTE: Play with the settings to get different results\n",
        "\n",
        "temperature = 0.8 #@param {type:\"slider\", min:0.1, max:1, step:0.1}\n",
        "show_stats = True #@param {type:\"boolean\"}\n",
        "\n",
        "#===================================================================\n",
        "print('=' * 70)\n",
        "print('Mini Muse Music Model Continuation Generator')\n",
        "print('=' * 70)\n",
        "\n",
        "print('Generation settings:')\n",
        "print('=' * 70)\n",
        "print('Number of prime tokens:', number_of_prime_tokens)\n",
        "print('Model temperature:', temperature)\n",
        "\n",
        "print('=' * 70)\n",
        "print('Generating...')\n",
        "\n",
        "inputs = train_data[iindex:iindex+number_of_prime_tokens+1]\n",
        "\n",
        "rand_seq = model.generate(torch.Tensor(inputs), \n",
        "                                          target_seq_length=1024,\n",
        "                                          temperature=temperature,\n",
        "                                          stop_token=512,\n",
        "                                          verbose=show_stats)\n",
        "  \n",
        "out1 = rand_seq[0].cpu().numpy().tolist()\n",
        "\n",
        "if len(out1) != 0:\n",
        "    \n",
        "    song = out1\n",
        "    song_f = []\n",
        "    time = 0\n",
        "    dur = 0\n",
        "    vel = 0\n",
        "    pitch = 0\n",
        "    channel = 0\n",
        "\n",
        "    son = []\n",
        "\n",
        "    song1 = []\n",
        "\n",
        "    for s in song:\n",
        "      if s > 127:\n",
        "        son.append(s)\n",
        "\n",
        "      else:\n",
        "        if len(son) == 4:\n",
        "          song1.append(son)\n",
        "        son = []\n",
        "        son.append(s)\n",
        "    \n",
        "    for s in song1:\n",
        "\n",
        "        channel = s[0] // 10\n",
        "\n",
        "        vel = (s[0] % 10) * 16\n",
        "\n",
        "        time += (s[1]-128) * 16\n",
        "            \n",
        "        dur = (s[2] - 256) * 32\n",
        "        \n",
        "        pitch = (s[3] - 384)\n",
        "                                  \n",
        "        song_f.append(['note', time, dur, channel, pitch, vel ])\n",
        "\n",
        "    detailed_stats = TMIDIX.Tegridy_SONG_to_MIDI_Converter(song_f,\n",
        "                                                        output_signature = 'Mini Muse',  \n",
        "                                                        output_file_name = '/content/Mini-Muse-Music-Composition', \n",
        "                                                        track_name='Project Los Angeles',\n",
        "                                                        list_of_MIDI_patches=[0, 24, 32, 40, 42, 46, 56, 71, 73, 0, 53, 0, 0, 0, 0, 0],\n",
        "                                                        number_of_ticks_per_quarter=500)\n",
        "\n",
        "    print('Done!')\n",
        "\n",
        "print('Displaying resulting composition...')\n",
        "fname = '/content/Mini-Muse-Music-Composition'\n",
        "\n",
        "pm = pretty_midi.PrettyMIDI(fname + '.mid')\n",
        "\n",
        "# Retrieve piano roll of the MIDI file\n",
        "piano_roll = pm.get_piano_roll()\n",
        "\n",
        "plt.figure(figsize=(14, 5))\n",
        "librosa.display.specshow(piano_roll, x_axis='time', y_axis='cqt_note', fmin=1, hop_length=160, sr=16000, cmap=plt.cm.hot)\n",
        "plt.title(fname)\n",
        "\n",
        "FluidSynth(\"/usr/share/sounds/sf2/FluidR3_GM.sf2\", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))\n",
        "Audio(str(fname + '.wav'), rate=16000)"
      ],
      "metadata": {
        "id": "nD-Iu_R0xbnx",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Auto-continue resulting composition\n",
        "\n",
        "#@markdown NOTE: This may or may not work well due to long-term structure decay\n",
        "number_of_continuation_blocks = 1 #@param {type:\"slider\", min:1, max:5, step:1}\n",
        "\n",
        "out2 = copy.deepcopy(out1)\n",
        "\n",
        "for i in range(number_of_continuation_blocks):\n",
        "\n",
        "  rand_seq = model.generate(torch.Tensor(out2[-64:]), \n",
        "                                            target_seq_length=1024,\n",
        "                                            temperature=temperature,\n",
        "                                            stop_token=512,\n",
        "                                            verbose=show_stats)\n",
        "    \n",
        "  out = rand_seq[0].cpu().numpy().tolist()\n",
        "\n",
        "  out2.extend(out[64:])\n",
        "\n",
        "if len(out2) != 0:\n",
        "    \n",
        "    song = out2\n",
        "    song_f = []\n",
        "    time = 0\n",
        "    dur = 0\n",
        "    vel = 0\n",
        "    pitch = 0\n",
        "    channel = 0\n",
        "\n",
        "    son = []\n",
        "\n",
        "    song1 = []\n",
        "\n",
        "    for s in song:\n",
        "      if s > 127:\n",
        "        son.append(s)\n",
        "\n",
        "      else:\n",
        "        if len(son) == 4:\n",
        "          song1.append(son)\n",
        "        son = []\n",
        "        son.append(s)\n",
        "    \n",
        "    for s in song1:\n",
        "\n",
        "        channel = s[0] // 10\n",
        "\n",
        "        vel = (s[0] % 10) * 16\n",
        "\n",
        "        time += (s[1]-128) * 16\n",
        "            \n",
        "        dur = (s[2] - 256) * 32\n",
        "        \n",
        "        pitch = (s[3] - 384)\n",
        "                                  \n",
        "        song_f.append(['note', time, dur, channel, pitch, vel ])\n",
        "\n",
        "    detailed_stats = TMIDIX.Tegridy_SONG_to_MIDI_Converter(song_f,\n",
        "                                                        output_signature = 'Mini Muse',  \n",
        "                                                        output_file_name = '/content/Mini-Muse-Music-Composition', \n",
        "                                                        track_name='Project Los Angeles',\n",
        "                                                        list_of_MIDI_patches=[0, 24, 32, 40, 42, 46, 56, 71, 73, 0, 53, 0, 0, 0, 0, 0],\n",
        "                                                        number_of_ticks_per_quarter=500)\n",
        "\n",
        "    print('Done!')\n",
        "\n",
        "print('Displaying resulting composition...')\n",
        "fname = '/content/Mini-Muse-Music-Composition'\n",
        "\n",
        "pm = pretty_midi.PrettyMIDI(fname + '.mid')\n",
        "\n",
        "# Retrieve piano roll of the MIDI file\n",
        "piano_roll = pm.get_piano_roll()\n",
        "\n",
        "plt.figure(figsize=(14, 5))\n",
        "librosa.display.specshow(piano_roll, x_axis='time', y_axis='cqt_note', fmin=1, hop_length=160, sr=16000, cmap=plt.cm.hot)\n",
        "plt.title(fname)\n",
        "\n",
        "FluidSynth(\"/usr/share/sounds/sf2/FluidR3_GM.sf2\", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))\n",
        "Audio(str(fname + '.wav'), rate=16000)"
      ],
      "metadata": {
        "cellView": "form",
        "id": "CHATK6H7kmhf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YzCMd94Tu_gz"
      },
      "source": [
        "# Congrats! You did it! :)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "Mini_Muse.ipynb",
      "private_outputs": true,
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.7"
    },
    "gpuClass": "standard",
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}