Gradio.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "210d1b87",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "62216909",
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "import os\n",
    "import xml.etree.ElementTree as et\n",
    "from pathlib import Path\n",
    "from zipfile import ZipFile\n",
    "import json\n",
    "import gradio as gr\n",
    "import pycountry\n",
    "from averell.utils import get_ids, CORPORA_SOURCES, CORPUS_NAMES\n",
    "from averell.core import export_corpora, get_corpora"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "82ba2880",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_ids(\"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "id": "2c4d3b2d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'name': 'Disco V2.1',\n",
       "  'properties': {'slug': 'disco2_1',\n",
       "   'license': 'CC-BY',\n",
       "   'language': 'es',\n",
       "   'reader': 'averell.readers.disco',\n",
       "   'url': 'https://github.com/pruizf/disco/archive/v2.1.zip',\n",
       "   'size': '22M',\n",
       "   'doc_quantity': 4088,\n",
       "   'word_quantity': 381539,\n",
       "   'granularity': ['stanza', 'line']}},\n",
       " {'name': 'Disco V3',\n",
       "  'properties': {'slug': 'disco3',\n",
       "   'license': 'CC-BY',\n",
       "   'language': 'es',\n",
       "   'reader': 'averell.readers.disco3',\n",
       "   'url': 'https://github.com/pruizf/disco/archive/v3.zip',\n",
       "   'size': '28M',\n",
       "   'doc_quantity': 4080,\n",
       "   'word_quantity': 377978,\n",
       "   'granularity': ['stanza', 'line']}},\n",
       " {'name': 'Sonetos Siglo de Oro',\n",
       "  'properties': {'slug': 'adso',\n",
       "   'license': 'CC-BY-NC 4.0',\n",
       "   'language': 'es',\n",
       "   'reader': 'averell.readers.sdo',\n",
       "   'url': 'https://github.com/bncolorado/CorpusSonetosSigloDeOro/archive/master.zip',\n",
       "   'size': '6.8M',\n",
       "   'doc_quantity': 5078,\n",
       "   'word_quantity': 466012,\n",
       "   'granularity': ['stanza', 'line']}},\n",
       " {'name': 'ADSO 100 poems corpus',\n",
       "  'properties': {'slug': 'adso100',\n",
       "   'license': 'CC-BY-NC 4.0',\n",
       "   'language': 'es',\n",
       "   'reader': 'averell.readers.sdo',\n",
       "   'url': 'https://github.com/linhd-postdata/adsoScansionSystem/releases/download/1.0.0/ADSO_gold_standard_100poems.zip',\n",
       "   'size': '128K',\n",
       "   'doc_quantity': 100,\n",
       "   'word_quantity': 9208,\n",
       "   'granularity': ['stanza', 'line']}},\n",
       " {'name': 'Poesía Lírica Castellana Siglo de Oro',\n",
       "  'properties': {'slug': 'plc',\n",
       "   'license': 'CC-BY-NC 4.0',\n",
       "   'language': 'es',\n",
       "   'reader': 'averell.readers.plsdo',\n",
       "   'url': 'https://github.com/bncolorado/CorpusGeneralPoesiaLiricaCastellanaDelSigloDeOro/archive/master.zip',\n",
       "   'size': '3.8M',\n",
       "   'doc_quantity': 475,\n",
       "   'word_quantity': 299402,\n",
       "   'granularity': ['stanza', 'line', 'word', 'syllable']}},\n",
       " {'name': 'Gongocorpus',\n",
       "  'properties': {'slug': 'gongo',\n",
       "   'license': 'CC-BY-NC-ND 3.0 FR',\n",
       "   'language': 'es',\n",
       "   'reader': 'averell.readers.gongocorpus',\n",
       "   'url': 'https://github.com/linhd-postdata/gongocorpus/archive/master.zip',\n",
       "   'size': '9.2M',\n",
       "   'doc_quantity': 481,\n",
       "   'word_quantity': 99079,\n",
       "   'granularity': ['stanza', 'line', 'word', 'syllable']}},\n",
       " {'name': 'Eighteenth Century Poetry Archive',\n",
       "  'properties': {'slug': 'ecpa',\n",
       "   'license': 'CC BY-SA 4.0',\n",
       "   'language': 'en',\n",
       "   'reader': 'averell.readers.ecpa',\n",
       "   'url': 'https://github.com/alhuber1502/ECPA/archive/master.zip',\n",
       "   'size': '2400M',\n",
       "   'doc_quantity': 3084,\n",
       "   'word_quantity': 2063668,\n",
       "   'granularity': ['stanza', 'line', 'word']}},\n",
       " {'name': 'For Better For Verse',\n",
       "  'properties': {'slug': '4b4v',\n",
       "   'license': 'Unknown',\n",
       "   'language': 'en',\n",
       "   'reader': 'averell.readers.forbetter4verse',\n",
       "   'url': 'https://github.com/waynegraham/for_better_for_verse/archive/master.zip',\n",
       "   'size': '39.5M',\n",
       "   'doc_quantity': 103,\n",
       "   'word_quantity': 41749,\n",
       "   'granularity': ['stanza', 'line']}},\n",
       " {'name': 'Métrique en Ligne',\n",
       "  'properties': {'slug': 'mel',\n",
       "   'license': 'Unknown',\n",
       "   'language': 'fr',\n",
       "   'reader': 'averell.readers.metriqueenligne',\n",
       "   'url': 'https://github.com/linhd-postdata/metrique-en-ligne/archive/master.zip',\n",
       "   'size': '183M',\n",
       "   'doc_quantity': 5081,\n",
       "   'word_quantity': 1850222,\n",
       "   'granularity': ['stanza', 'line']}},\n",
       " {'name': 'Biblioteca Italiana',\n",
       "  'properties': {'slug': 'bibit',\n",
       "   'license': 'Unknown',\n",
       "   'language': 'it',\n",
       "   'reader': 'averell.readers.bibliotecaitaliana',\n",
       "   'url': 'https://github.com/linhd-postdata/biblioteca_italiana/archive/master.zip',\n",
       "   'size': '242M',\n",
       "   'doc_quantity': 25341,\n",
       "   'word_quantity': 7121246,\n",
       "   'granularity': ['stanza', 'line', 'word']}},\n",
       " {'name': 'Corpus of Czech Verse',\n",
       "  'properties': {'slug': 'czverse',\n",
       "   'license': 'CC-BY-SA',\n",
       "   'language': 'cs',\n",
       "   'reader': 'averell.readers.czechverse',\n",
       "   'url': 'https://github.com/versotym/corpusCzechVerse/archive/refs/heads/master.zip',\n",
       "   'size': '4100M',\n",
       "   'doc_quantity': 66428,\n",
       "   'word_quantity': 12636867,\n",
       "   'granularity': ['stanza', 'line', 'word']}},\n",
       " {'name': 'Stichotheque Portuguese',\n",
       "  'properties': {'slug': 'stichopt',\n",
       "   'license': 'Unkwown',\n",
       "   'language': 'pt',\n",
       "   'reader': 'averell.readers.stichotheque',\n",
       "   'url': 'https://gitlab.com/stichotheque/stichotheque-pt/-/archive/master/stichotheque-pt-master.zip',\n",
       "   'size': '11.8M',\n",
       "   'doc_quantity': 1702,\n",
       "   'word_quantity': 168411,\n",
       "   'granularity': ['stanza', 'line']}}]"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CORPORA_SOURCES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5ec7d935",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_available_languages(sources):\n",
    "    available_languages = []\n",
    "    for c in sources:\n",
    "        lang = c[\"properties\"][\"language\"]\n",
    "        if lang not in available_languages:\n",
    "            available_languages.append(lang)\n",
    "    return available_languages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d9663965",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_corpus_language(sources, lang):\n",
    "    corpora_sources = copy.deepcopy(sources)\n",
    "    filtered_corpora = [c for c in corpora_sources if c[\"properties\"][\"language\"] == lang]\n",
    "    return filtered_corpora"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "id": "a2ae8486",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "adso100\n",
      "tmp/\n",
      "TEI\n",
      "adso100\n",
      "tmp/\n",
      "TEI\n",
      "adso100\n",
      "disco2_1\n",
      "tmp/\n",
      "tmp/\n",
      "TEI\n",
      "gongo\n",
      "tmp/\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:root:'word' granularity not found on 'Disco V2.1' properties\n",
      "ERROR:root:'word' granularity not found on 'Disco V3' properties\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp/\n",
      "granularity not found\n",
      "granularity not found\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                   | 0/1 [00:00<?, ?it/s]\n",
      "adso.zip: 0.00B [00:00, ?B/s]\u001b[A\n",
      "adso.zip: 8.00kB [00:02, 3.11kB/s]\u001b[A\n",
      "adso.zip: 40.0kB [00:02, 19.4kB/s]\u001b[A\n",
      "adso.zip: 296kB [00:02, 194kB/s]  \u001b[A\n",
      "adso.zip: 600kB [00:02, 448kB/s]\u001b[A\n",
      "adso.zip: 984kB [00:03, 834kB/s]\u001b[A\n",
      "adso.zip: 1.41MB [00:03, 1.35MB/s]\u001b[A\n",
      "adso.zip: 1.89MB [00:03, 1.95MB/s]\u001b[A\n",
      "adso.zip: 2.44MB [00:03, 2.66MB/s]\u001b[A\n",
      "adso.zip: 3.08MB [00:03, 3.51MB/s]\u001b[A\n",
      "adso.zip: 3.97MB [00:03, 4.79MB/s]\u001b[A\n",
      "adso.zip: 4.85MB [00:03, 5.89MB/s]\u001b[A\n",
      "adso.zip: 6.76MB [00:03, 1.83MB/s]\u001b[A\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 1/1 [00:04<00:00,  4.02s/it]\n",
      "INFO:root:Downloaded Sonetos Siglo de Oro corpus\n",
      "ERROR:root:'word' granularity not found on 'Sonetos Siglo de Oro' properties\n",
      "ERROR:root:'word' granularity not found on 'ADSO 100 poems corpus' properties\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "granularity not found\n",
      "granularity not found\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                   | 0/1 [00:00<?, ?it/s]\n",
      "plc.zip: 0.00B [00:00, ?B/s]\u001b[A\n",
      "plc.zip: 8.00kB [00:00, 12.7kB/s]\u001b[A\n",
      "plc.zip: 200kB [00:00, 361kB/s]  \u001b[A\n",
      "plc.zip: 992kB [00:00, 1.93MB/s]\u001b[A\n",
      "plc.zip: 1.77MB [00:00, 3.39MB/s]\u001b[A\n",
      "plc.zip: 2.34MB [00:01, 3.89MB/s]\u001b[A\n",
      "plc.zip: 3.13MB [00:01, 4.94MB/s]\u001b[A\n",
      "plc.zip: 3.80MB [00:01, 3.09MB/s]\u001b[A\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.35s/it]\n",
      "INFO:root:Downloaded Poesía Lírica Castellana Siglo de Oro corpus\n",
      "  0%|                                                                                   | 0/1 [00:00<?, ?it/s]\n",
      "gongo.zip: 0.00B [00:00, ?B/s]\u001b[A\n",
      "gongo.zip: 8.00kB [00:02, 3.09kB/s]\u001b[A\n",
      "gongo.zip: 200kB [00:02, 103kB/s]  \u001b[A\n",
      "gongo.zip: 808kB [00:02, 517kB/s]\u001b[A\n",
      "gongo.zip: 1.08MB [00:02, 739kB/s]\u001b[A\n",
      "gongo.zip: 1.39MB [00:03, 1.02MB/s]\u001b[A\n",
      "gongo.zip: 1.68MB [00:03, 1.29MB/s]\u001b[A\n",
      "gongo.zip: 1.82MB [00:03, 1.32MB/s]\u001b[A\n",
      "gongo.zip: 2.23MB [00:03, 1.88MB/s]\u001b[A\n",
      "gongo.zip: 3.01MB [00:03, 884kB/s] \u001b[A\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 1/1 [00:03<00:00,  3.67s/it]\n",
      "INFO:root:Downloaded Gongocorpus corpus\n",
      "  0%|                                                                                   | 0/1 [00:00<?, ?it/s]\n",
      "ecpa.zip: 0.00B [00:00, ?B/s]\u001b[A\n",
      "ecpa.zip: 8.00kB [00:01, 6.16kB/s]\u001b[A\n",
      "ecpa.zip: 200kB [00:01, 195kB/s]  \u001b[A\n",
      "ecpa.zip: 1.84MB [00:01, 2.26MB/s]\u001b[A\n",
      "ecpa.zip: 2.08MB [00:02, 1.04MB/s]\u001b[A\n",
      "ecpa.zip: 5.58MB [00:02, 4.16MB/s]\u001b[A\n",
      "ecpa.zip: 8.48MB [00:02, 7.06MB/s]\u001b[A\n",
      "ecpa.zip: 11.3MB [00:02, 10.2MB/s]\u001b[A\n",
      "ecpa.zip: 14.0MB [00:02, 13.1MB/s]\u001b[A\n",
      "ecpa.zip: 16.4MB [00:02, 15.4MB/s]\u001b[A\n",
      "ecpa.zip: 19.3MB [00:03, 17.8MB/s]\u001b[A\n",
      "ecpa.zip: 21.5MB [00:03, 19.0MB/s]\u001b[A\n",
      "ecpa.zip: 22.4MB [00:03, 14.7MB/s]\u001b[A\n",
      "ecpa.zip: 24.3MB [00:03, 15.6MB/s]\u001b[A\n",
      "ecpa.zip: 27.9MB [00:03, 21.0MB/s]\u001b[A\n",
      "ecpa.zip: 30.5MB [00:03, 22.6MB/s]\u001b[A\n",
      "ecpa.zip: 33.0MB [00:03, 23.5MB/s]\u001b[A\n",
      "ecpa.zip: 35.4MB [00:03, 23.9MB/s]\u001b[A\n",
      "ecpa.zip: 37.6MB [00:03, 23.9MB/s]\u001b[A\n",
      "ecpa.zip: 40.8MB [00:04, 26.6MB/s]\u001b[A\n",
      "ecpa.zip: 43.5MB [00:04, 27.1MB/s]\u001b[A\n",
      "ecpa.zip: 46.1MB [00:04, 25.8MB/s]\u001b[A\n",
      "ecpa.zip: 48.9MB [00:04, 26.7MB/s]\u001b[A\n",
      "ecpa.zip: 51.5MB [00:04, 26.9MB/s]\u001b[A\n",
      "ecpa.zip: 54.4MB [00:04, 27.8MB/s]\u001b[A\n",
      "ecpa.zip: 57.1MB [00:04, 28.0MB/s]\u001b[A\n",
      "ecpa.zip: 58.2MB [00:04, 23.0MB/s]\u001b[A\n",
      "ecpa.zip: 60.1MB [00:04, 22.1MB/s]\u001b[A\n",
      "ecpa.zip: 60.6MB [00:05, 5.19MB/s]\u001b[A\n",
      "ecpa.zip: 62.0MB [00:05, 6.27MB/s]\u001b[A\n",
      "ecpa.zip: 64.9MB [00:05, 9.46MB/s]\u001b[A\n",
      "ecpa.zip: 65.9MB [00:06, 9.66MB/s]\u001b[A\n",
      "ecpa.zip: 66.4MB [00:06, 8.44MB/s]\u001b[A\n",
      "ecpa.zip: 66.9MB [00:06, 7.56MB/s]\u001b[A\n",
      "ecpa.zip: 67.8MB [00:06, 7.58MB/s]\u001b[A\n",
      "ecpa.zip: 70.2MB [00:06, 11.7MB/s]\u001b[A\n",
      "ecpa.zip: 71.1MB [00:06, 11.1MB/s]\u001b[A\n",
      "ecpa.zip: 71.7MB [00:06, 10.0MB/s]\u001b[A\n",
      "ecpa.zip: 72.4MB [00:06, 9.06MB/s]\u001b[A\n",
      "ecpa.zip: 73.1MB [00:06, 8.49MB/s]\u001b[A\n",
      "ecpa.zip: 73.7MB [00:07, 7.84MB/s]\u001b[A\n",
      "ecpa.zip: 74.3MB [00:07, 7.42MB/s]\u001b[A\n",
      "ecpa.zip: 74.9MB [00:07, 7.28MB/s]\u001b[A\n",
      "ecpa.zip: 75.7MB [00:07, 7.34MB/s]\u001b[A\n",
      "ecpa.zip: 76.4MB [00:07, 7.44MB/s]\u001b[A\n",
      "ecpa.zip: 76.6MB [00:07, 5.22MB/s]\u001b[A\n",
      "ecpa.zip: 78.1MB [00:07, 8.17MB/s]\u001b[A\n",
      "ecpa.zip: 78.8MB [00:07, 7.76MB/s]\u001b[A\n",
      "ecpa.zip: 79.5MB [00:07, 7.57MB/s]\u001b[A\n",
      "ecpa.zip: 80.1MB [00:07, 7.25MB/s]\u001b[A\n",
      "ecpa.zip: 80.6MB [00:08, 6.69MB/s]\u001b[A\n",
      "ecpa.zip: 81.0MB [00:08, 5.84MB/s]\u001b[A\n",
      "ecpa.zip: 81.4MB [00:08, 5.23MB/s]\u001b[A\n",
      "ecpa.zip: 81.7MB [00:08, 4.87MB/s]\u001b[A\n",
      "ecpa.zip: 82.1MB [00:08, 4.64MB/s]\u001b[A\n",
      "ecpa.zip: 82.5MB [00:08, 4.50MB/s]\u001b[A\n",
      "ecpa.zip: 83.0MB [00:08, 4.44MB/s]\u001b[A\n",
      "ecpa.zip: 83.4MB [00:08, 4.49MB/s]\u001b[A\n",
      "ecpa.zip: 84.4MB [00:08, 6.11MB/s]\u001b[A\n",
      "ecpa.zip: 84.7MB [00:08, 5.07MB/s]\u001b[A\n",
      "ecpa.zip: 86.2MB [00:09, 7.77MB/s]\u001b[A\n",
      "ecpa.zip: 86.5MB [00:09, 2.64MB/s]\u001b[A\n",
      "ecpa.zip: 89.1MB [00:09, 6.12MB/s]\u001b[A\n",
      "ecpa.zip: 90.9MB [00:09, 8.16MB/s]\u001b[A\n",
      "ecpa.zip: 93.8MB [00:10, 12.5MB/s]\u001b[A\n",
      "ecpa.zip: 94.8MB [00:10, 11.7MB/s]\u001b[A\n",
      "ecpa.zip: 95.2MB [00:10, 9.49MB/s]\u001b[A\n",
      "ecpa.zip: 95.8MB [00:10, 8.52MB/s]\u001b[A\n",
      "ecpa.zip: 96.2MB [00:10, 7.09MB/s]\u001b[A\n",
      "ecpa.zip: 97.0MB [00:10, 7.21MB/s]\u001b[A\n",
      "ecpa.zip: 97.6MB [00:10, 6.80MB/s]\u001b[A\n",
      "ecpa.zip: 98.1MB [00:10, 6.28MB/s]\u001b[A\n",
      "ecpa.zip: 98.5MB [00:10, 5.03MB/s]\u001b[A\n",
      "ecpa.zip: 99.3MB [00:11, 5.79MB/s]\u001b[A\n",
      "ecpa.zip: 99.9MB [00:11, 5.91MB/s]\u001b[A\n",
      "ecpa.zip: 100MB [00:11, 4.88MB/s] \u001b[A\n",
      "ecpa.zip: 101MB [00:11, 5.67MB/s]\u001b[A\n",
      "ecpa.zip: 102MB [00:11, 5.55MB/s]\u001b[A\n",
      "ecpa.zip: 102MB [00:11, 4.71MB/s]\u001b[A\n",
      "ecpa.zip: 103MB [00:11, 6.02MB/s]\u001b[A\n",
      "ecpa.zip: 104MB [00:11, 6.65MB/s]\u001b[A\n",
      "ecpa.zip: 104MB [00:12, 5.34MB/s]\u001b[A\n",
      "ecpa.zip: 105MB [00:12, 4.33MB/s]\u001b[A\n",
      "ecpa.zip: 105MB [00:12, 3.49MB/s]\u001b[A\n",
      "ecpa.zip: 105MB [00:12, 2.75MB/s]\u001b[A\n",
      "ecpa.zip: 105MB [00:12, 2.76MB/s]\u001b[A\n",
      "ecpa.zip: 105MB [00:12, 2.62MB/s]\u001b[A\n",
      "ecpa.zip: 105MB [00:12, 1.96MB/s]\u001b[A\n",
      "ecpa.zip: 105MB [00:12, 1.73MB/s]\u001b[A\n",
      "ecpa.zip: 106MB [00:12, 1.55MB/s]\u001b[A\n",
      "ecpa.zip: 106MB [00:12, 1.45MB/s]\u001b[A\n",
      "ecpa.zip: 106MB [00:13, 1.17MB/s]\u001b[A\n",
      "ecpa.zip: 106MB [00:13, 1.28MB/s]\u001b[A\n",
      "ecpa.zip: 106MB [00:13, 1.15MB/s]\u001b[A\n",
      "ecpa.zip: 106MB [00:13, 1.21MB/s]\u001b[A\n",
      "ecpa.zip: 106MB [00:13, 1.19MB/s]\u001b[A\n",
      "ecpa.zip: 106MB [00:13, 1.25MB/s]\u001b[A\n",
      "ecpa.zip: 107MB [00:13, 1.76MB/s]\u001b[A\n",
      "ecpa.zip: 107MB [00:13, 2.40MB/s]\u001b[A\n",
      "ecpa.zip: 107MB [00:13, 2.32MB/s]\u001b[A\n",
      "ecpa.zip: 107MB [00:14, 2.08MB/s]\u001b[A\n",
      "ecpa.zip: 108MB [00:14, 1.99MB/s]\u001b[A\n",
      "ecpa.zip: 108MB [00:14, 2.05MB/s]\u001b[A\n",
      "ecpa.zip: 108MB [00:14, 1.90MB/s]\u001b[A\n",
      "ecpa.zip: 108MB [00:14, 2.00MB/s]\u001b[A\n",
      "ecpa.zip: 108MB [00:14, 1.84MB/s]\u001b[A\n",
      "ecpa.zip: 108MB [00:14, 1.56MB/s]\u001b[A\n",
      "ecpa.zip: 109MB [00:14, 1.46MB/s]\u001b[A\n",
      "ecpa.zip: 109MB [00:14, 1.70MB/s]\u001b[A\n",
      "ecpa.zip: 109MB [00:15, 2.13MB/s]\u001b[A\n",
      "ecpa.zip: 110MB [00:15, 4.03MB/s]\u001b[A\n",
      "ecpa.zip: 111MB [00:15, 4.30MB/s]\u001b[A\n",
      "ecpa.zip: 111MB [00:15, 4.01MB/s]\u001b[A\n",
      "ecpa.zip: 111MB [00:15, 3.62MB/s]\u001b[A\n",
      "ecpa.zip: 111MB [00:15, 3.02MB/s]\u001b[A\n",
      "ecpa.zip: 112MB [00:15, 2.77MB/s]\u001b[A\n",
      "ecpa.zip: 112MB [00:15, 3.32MB/s]\u001b[A\n",
      "ecpa.zip: 112MB [00:15, 2.76MB/s]\u001b[A\n",
      "ecpa.zip: 112MB [00:16, 2.15MB/s]\u001b[A\n",
      "ecpa.zip: 112MB [00:16, 1.83MB/s]\u001b[A\n",
      "ecpa.zip: 113MB [00:16, 1.65MB/s]\u001b[A\n",
      "ecpa.zip: 113MB [00:16, 1.34MB/s]\u001b[A\n",
      "ecpa.zip: 113MB [00:16, 1.54MB/s]\u001b[A\n",
      "ecpa.zip: 113MB [00:16, 1.65MB/s]\u001b[A\n",
      "ecpa.zip: 113MB [00:16, 1.89MB/s]\u001b[A\n",
      "ecpa.zip: 113MB [00:16, 2.02MB/s]\u001b[A\n",
      "ecpa.zip: 114MB [00:16, 2.12MB/s]\u001b[A\n",
      "ecpa.zip: 114MB [00:16, 2.35MB/s]\u001b[A\n",
      "ecpa.zip: 114MB [00:17, 2.75MB/s]\u001b[A\n",
      "ecpa.zip: 115MB [00:17, 3.29MB/s]\u001b[A\n",
      "ecpa.zip: 115MB [00:17, 3.34MB/s]\u001b[A\n",
      "ecpa.zip: 115MB [00:17, 3.11MB/s]\u001b[A\n",
      "ecpa.zip: 116MB [00:17, 2.56MB/s]\u001b[A\n",
      "ecpa.zip: 116MB [00:17, 2.51MB/s]\u001b[A\n",
      "ecpa.zip: 116MB [00:17, 3.04MB/s]\u001b[A\n",
      "ecpa.zip: 117MB [00:17, 3.20MB/s]\u001b[A\n",
      "ecpa.zip: 117MB [00:17, 2.94MB/s]\u001b[A\n",
      "ecpa.zip: 117MB [00:18, 3.34MB/s]\u001b[A\n",
      "ecpa.zip: 118MB [00:18, 3.30MB/s]\u001b[A\n",
      "ecpa.zip: 118MB [00:18, 3.15MB/s]\u001b[A\n",
      "ecpa.zip: 118MB [00:18, 2.65MB/s]\u001b[A\n",
      "ecpa.zip: 118MB [00:18, 2.18MB/s]\u001b[A\n",
      "ecpa.zip: 119MB [00:18, 2.84MB/s]\u001b[A\n",
      "ecpa.zip: 120MB [00:18, 3.76MB/s]\u001b[A\n",
      "ecpa.zip: 120MB [00:18, 3.59MB/s]\u001b[A\n",
      "ecpa.zip: 120MB [00:19, 3.34MB/s]\u001b[A\n",
      "ecpa.zip: 121MB [00:19, 3.48MB/s]\u001b[A\n",
      "ecpa.zip: 121MB [00:19, 3.39MB/s]\u001b[A\n",
      "ecpa.zip: 121MB [00:19, 2.74MB/s]\u001b[A\n",
      "ecpa.zip: 121MB [00:19, 2.47MB/s]\u001b[A\n",
      "ecpa.zip: 121MB [00:19, 2.37MB/s]\u001b[A\n",
      "ecpa.zip: 122MB [00:19, 2.77MB/s]\u001b[A\n",
      "ecpa.zip: 122MB [00:19, 2.88MB/s]\u001b[A\n",
      "ecpa.zip: 123MB [00:19, 4.01MB/s]\u001b[A\n",
      "ecpa.zip: 123MB [00:19, 4.80MB/s]\u001b[A\n",
      "ecpa.zip: 124MB [00:20, 5.49MB/s]\u001b[A\n",
      "ecpa.zip: 125MB [00:20, 6.11MB/s]\u001b[A\n",
      "ecpa.zip: 125MB [00:20, 5.33MB/s]\u001b[A\n",
      "ecpa.zip: 126MB [00:20, 4.70MB/s]\u001b[A\n",
      "ecpa.zip: 126MB [00:20, 4.73MB/s]\u001b[A\n",
      "ecpa.zip: 126MB [00:20, 3.72MB/s]\u001b[A\n",
      "ecpa.zip: 127MB [00:20, 4.23MB/s]\u001b[A\n",
      "ecpa.zip: 127MB [00:20, 4.14MB/s]\u001b[A\n",
      "ecpa.zip: 127MB [00:20, 3.91MB/s]\u001b[A\n",
      "ecpa.zip: 128MB [00:21, 3.93MB/s]\u001b[A\n",
      "ecpa.zip: 128MB [00:21, 4.93MB/s]\u001b[A\n",
      "ecpa.zip: 129MB [00:21, 4.33MB/s]\u001b[A\n",
      "ecpa.zip: 129MB [00:21, 3.98MB/s]\u001b[A\n",
      "ecpa.zip: 129MB [00:21, 3.72MB/s]\u001b[A\n",
      "ecpa.zip: 130MB [00:21, 3.18MB/s]\u001b[A\n",
      "ecpa.zip: 130MB [00:21, 2.90MB/s]\u001b[A\n",
      "ecpa.zip: 130MB [00:21, 3.35MB/s]\u001b[A\n",
      "ecpa.zip: 131MB [00:21, 3.14MB/s]\u001b[A\n",
      "ecpa.zip: 131MB [00:21, 3.35MB/s]\u001b[A\n",
      "ecpa.zip: 131MB [00:22, 3.36MB/s]\u001b[A\n",
      "ecpa.zip: 132MB [00:22, 3.22MB/s]\u001b[A\n",
      "ecpa.zip: 132MB [00:22, 3.35MB/s]\u001b[A\n",
      "ecpa.zip: 132MB [00:22, 3.01MB/s]\u001b[A\n",
      "ecpa.zip: 132MB [00:22, 2.81MB/s]\u001b[A\n",
      "ecpa.zip: 133MB [00:22, 2.71MB/s]\u001b[A\n",
      "ecpa.zip: 133MB [00:22, 2.63MB/s]\u001b[A\n",
      "ecpa.zip: 133MB [00:22, 2.65MB/s]\u001b[A\n",
      "ecpa.zip: 133MB [00:22, 2.32MB/s]\u001b[A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ecpa.zip: 134MB [00:23, 2.33MB/s]\u001b[A\n",
      "ecpa.zip: 134MB [00:23, 2.60MB/s]\u001b[A\n",
      "ecpa.zip: 134MB [00:23, 2.61MB/s]\u001b[A\n",
      "ecpa.zip: 134MB [00:23, 2.46MB/s]\u001b[A\n",
      "ecpa.zip: 135MB [00:23, 2.85MB/s]\u001b[A\n",
      "ecpa.zip: 135MB [00:23, 2.68MB/s]\u001b[A\n",
      "ecpa.zip: 135MB [00:23, 2.43MB/s]\u001b[A\n",
      "ecpa.zip: 135MB [00:23, 2.07MB/s]\u001b[A\n",
      "ecpa.zip: 136MB [00:23, 2.30MB/s]\u001b[A\n",
      "ecpa.zip: 136MB [00:23, 2.81MB/s]\u001b[A\n",
      "ecpa.zip: 136MB [00:24, 2.87MB/s]\u001b[A\n",
      "ecpa.zip: 136MB [00:24, 2.18MB/s]\u001b[A\n",
      "ecpa.zip: 137MB [00:24, 1.82MB/s]\u001b[A\n",
      "ecpa.zip: 137MB [00:24, 2.27MB/s]\u001b[A\n",
      "ecpa.zip: 137MB [00:24, 2.97MB/s]\u001b[A\n",
      "ecpa.zip: 138MB [00:24, 2.75MB/s]\u001b[A\n",
      "ecpa.zip: 138MB [00:24, 2.74MB/s]\u001b[A\n",
      "ecpa.zip: 138MB [00:24, 3.70MB/s]\u001b[A\n",
      "ecpa.zip: 139MB [00:24, 4.27MB/s]\u001b[A\n",
      "ecpa.zip: 139MB [00:25, 4.32MB/s]\u001b[A\n",
      "ecpa.zip: 140MB [00:25, 4.31MB/s]\u001b[A\n",
      "ecpa.zip: 140MB [00:25, 4.50MB/s]\u001b[A\n",
      "ecpa.zip: 141MB [00:25, 4.69MB/s]\u001b[A\n",
      "ecpa.zip: 141MB [00:25, 4.55MB/s]\u001b[A\n",
      "ecpa.zip: 142MB [00:25, 4.84MB/s]\u001b[A\n",
      "ecpa.zip: 142MB [00:25, 3.28MB/s]\u001b[A\n",
      "ecpa.zip: 143MB [00:25, 5.91MB/s]\u001b[A\n",
      "ecpa.zip: 144MB [00:25, 5.62MB/s]\u001b[A\n",
      "ecpa.zip: 144MB [00:26, 4.78MB/s]\u001b[A\n",
      "ecpa.zip: 144MB [00:26, 4.07MB/s]\u001b[A\n",
      "ecpa.zip: 145MB [00:26, 5.10MB/s]\u001b[A\n",
      "ecpa.zip: 146MB [00:26, 5.46MB/s]\u001b[A\n",
      "ecpa.zip: 147MB [00:26, 5.51MB/s]\u001b[A\n",
      "ecpa.zip: 147MB [00:26, 4.96MB/s]\u001b[A\n",
      "ecpa.zip: 147MB [00:26, 4.00MB/s]\u001b[A\n",
      "ecpa.zip: 147MB [00:26, 3.15MB/s]\u001b[A\n",
      "ecpa.zip: 148MB [00:26, 2.95MB/s]\u001b[A\n",
      "ecpa.zip: 148MB [00:27, 2.94MB/s]\u001b[A\n",
      "ecpa.zip: 148MB [00:27, 2.79MB/s]\u001b[A\n",
      "ecpa.zip: 148MB [00:27, 2.97MB/s]\u001b[A\n",
      "ecpa.zip: 149MB [00:27, 3.41MB/s]\u001b[A\n",
      "ecpa.zip: 149MB [00:27, 3.21MB/s]\u001b[A\n",
      "ecpa.zip: 149MB [00:27, 3.05MB/s]\u001b[A\n",
      "ecpa.zip: 150MB [00:27, 2.97MB/s]\u001b[A\n",
      "ecpa.zip: 150MB [00:27, 4.06MB/s]\u001b[A\n",
      "ecpa.zip: 151MB [00:27, 4.55MB/s]\u001b[A\n",
      "ecpa.zip: 151MB [00:28, 3.39MB/s]\u001b[A\n",
      "ecpa.zip: 151MB [00:28, 3.06MB/s]\u001b[A\n",
      "ecpa.zip: 152MB [00:28, 2.96MB/s]\u001b[A\n",
      "ecpa.zip: 153MB [00:28, 4.29MB/s]\u001b[A\n",
      "ecpa.zip: 153MB [00:28, 5.09MB/s]\u001b[A\n",
      "ecpa.zip: 154MB [00:28, 4.80MB/s]\u001b[A\n",
      "ecpa.zip: 154MB [00:28, 4.52MB/s]\u001b[A\n",
      "ecpa.zip: 154MB [00:28, 4.08MB/s]\u001b[A\n",
      "ecpa.zip: 155MB [00:29, 4.24MB/s]\u001b[A\n",
      "ecpa.zip: 155MB [00:29, 3.87MB/s]\u001b[A\n",
      "ecpa.zip: 156MB [00:29, 4.40MB/s]\u001b[A\n",
      "ecpa.zip: 157MB [00:29, 4.77MB/s]\u001b[A\n",
      "ecpa.zip: 157MB [00:29, 4.91MB/s]\u001b[A\n",
      "ecpa.zip: 158MB [00:29, 5.28MB/s]\u001b[A\n",
      "ecpa.zip: 158MB [00:29, 5.27MB/s]\u001b[A\n",
      "ecpa.zip: 159MB [00:29, 5.98MB/s]\u001b[A\n",
      "ecpa.zip: 160MB [00:29, 5.10MB/s]\u001b[A\n",
      "ecpa.zip: 160MB [00:30, 5.38MB/s]\u001b[A\n",
      "ecpa.zip: 161MB [00:30, 5.16MB/s]\u001b[A\n",
      "ecpa.zip: 161MB [00:30, 4.98MB/s]\u001b[A\n",
      "ecpa.zip: 162MB [00:30, 4.14MB/s]\u001b[A\n",
      "ecpa.zip: 162MB [00:30, 4.89MB/s]\u001b[A\n",
      "ecpa.zip: 163MB [00:30, 4.08MB/s]\u001b[A\n",
      "ecpa.zip: 163MB [00:30, 3.96MB/s]\u001b[A\n",
      "ecpa.zip: 163MB [00:30, 3.35MB/s]\u001b[A\n",
      "ecpa.zip: 163MB [00:31, 2.90MB/s]\u001b[A\n",
      "ecpa.zip: 163MB [00:31, 2.31MB/s]\u001b[A\n",
      "ecpa.zip: 164MB [00:31, 1.86MB/s]\u001b[A\n",
      "ecpa.zip: 164MB [00:31, 1.93MB/s]\u001b[A\n",
      "ecpa.zip: 164MB [00:31, 1.94MB/s]\u001b[A\n",
      "ecpa.zip: 164MB [00:31, 1.90MB/s]\u001b[A\n",
      "ecpa.zip: 164MB [00:31, 1.78MB/s]\u001b[A\n",
      "ecpa.zip: 165MB [00:31, 1.90MB/s]\u001b[A\n",
      "ecpa.zip: 165MB [00:31, 1.67MB/s]\u001b[A\n",
      "ecpa.zip: 165MB [00:31, 1.88MB/s]\u001b[A\n",
      "ecpa.zip: 165MB [00:32, 2.37MB/s]\u001b[A\n",
      "ecpa.zip: 165MB [00:32, 2.03MB/s]\u001b[A\n",
      "ecpa.zip: 166MB [00:32, 2.87MB/s]\u001b[A\n",
      "ecpa.zip: 166MB [00:32, 3.11MB/s]\u001b[A\n",
      "ecpa.zip: 167MB [00:32, 4.10MB/s]\u001b[A\n",
      "ecpa.zip: 167MB [00:32, 4.91MB/s]\u001b[A\n",
      "ecpa.zip: 168MB [00:32, 4.75MB/s]\u001b[A\n",
      "ecpa.zip: 168MB [00:32, 4.23MB/s]\u001b[A\n",
      "ecpa.zip: 169MB [00:32, 4.40MB/s]\u001b[A\n",
      "ecpa.zip: 169MB [00:33, 4.21MB/s]\u001b[A\n",
      "ecpa.zip: 170MB [00:33, 5.11MB/s]\u001b[A\n",
      "ecpa.zip: 170MB [00:33, 4.38MB/s]\u001b[A\n",
      "ecpa.zip: 171MB [00:33, 5.77MB/s]\u001b[A\n",
      "ecpa.zip: 172MB [00:33, 5.75MB/s]\u001b[A\n",
      "ecpa.zip: 172MB [00:33, 5.17MB/s]\u001b[A\n",
      "ecpa.zip: 173MB [00:33, 4.81MB/s]\u001b[A\n",
      "ecpa.zip: 173MB [00:33, 5.30MB/s]\u001b[A\n",
      "ecpa.zip: 173MB [00:33, 4.50MB/s]\u001b[A\n",
      "ecpa.zip: 174MB [00:33, 4.40MB/s]\u001b[A\n",
      "ecpa.zip: 174MB [00:34, 4.12MB/s]\u001b[A\n",
      "ecpa.zip: 175MB [00:34, 4.53MB/s]\u001b[A\n",
      "ecpa.zip: 175MB [00:34, 4.35MB/s]\u001b[A\n",
      "ecpa.zip: 176MB [00:34, 4.36MB/s]\u001b[A\n",
      "ecpa.zip: 176MB [00:34, 4.83MB/s]\u001b[A\n",
      "ecpa.zip: 177MB [00:34, 4.57MB/s]\u001b[A\n",
      "ecpa.zip: 177MB [00:34, 4.59MB/s]\u001b[A\n",
      "ecpa.zip: 178MB [00:34, 4.31MB/s]\u001b[A\n",
      "ecpa.zip: 178MB [00:34, 4.25MB/s]\u001b[A\n",
      "ecpa.zip: 179MB [00:35, 4.92MB/s]\u001b[A\n",
      "ecpa.zip: 179MB [00:35, 5.01MB/s]\u001b[A\n",
      "ecpa.zip: 180MB [00:35, 4.93MB/s]\u001b[A\n",
      "ecpa.zip: 180MB [00:35, 5.36MB/s]\u001b[A\n",
      "ecpa.zip: 181MB [00:35, 5.11MB/s]\u001b[A\n",
      "ecpa.zip: 181MB [00:35, 5.11MB/s]\u001b[A\n",
      "ecpa.zip: 182MB [00:35, 5.30MB/s]\u001b[A\n",
      "ecpa.zip: 182MB [00:35, 4.16MB/s]\u001b[A\n",
      "ecpa.zip: 182MB [00:35, 3.40MB/s]\u001b[A\n",
      "ecpa.zip: 183MB [00:36, 2.97MB/s]\u001b[A\n",
      "ecpa.zip: 183MB [00:36, 1.93MB/s]\u001b[A\n",
      "ecpa.zip: 183MB [00:36, 2.60MB/s]\u001b[A\n",
      "ecpa.zip: 183MB [00:36, 1.96MB/s]\u001b[A\n",
      "ecpa.zip: 183MB [00:36, 1.82MB/s]\u001b[A\n",
      "ecpa.zip: 184MB [00:36, 1.76MB/s]\u001b[A\n",
      "ecpa.zip: 184MB [00:36, 1.66MB/s]\u001b[A\n",
      "ecpa.zip: 184MB [00:36, 2.88MB/s]\u001b[A\n",
      "ecpa.zip: 185MB [00:37, 3.54MB/s]\u001b[A\n",
      "ecpa.zip: 185MB [00:37, 3.84MB/s]\u001b[A\n",
      "ecpa.zip: 186MB [00:37, 4.66MB/s]\u001b[A\n",
      "ecpa.zip: 187MB [00:37, 4.90MB/s]\u001b[A\n",
      "ecpa.zip: 187MB [00:37, 4.88MB/s]\u001b[A\n",
      "ecpa.zip: 188MB [00:37, 5.44MB/s]\u001b[A\n",
      "ecpa.zip: 189MB [00:37, 6.05MB/s]\u001b[A\n",
      "ecpa.zip: 189MB [00:37, 5.44MB/s]\u001b[A\n",
      "ecpa.zip: 190MB [00:38, 4.72MB/s]\u001b[A\n",
      "ecpa.zip: 190MB [00:38, 5.26MB/s]\u001b[A\n",
      "ecpa.zip: 191MB [00:38, 4.62MB/s]\u001b[A\n",
      "ecpa.zip: 191MB [00:38, 3.82MB/s]\u001b[A\n",
      "ecpa.zip: 192MB [00:38, 4.46MB/s]\u001b[A\n",
      "ecpa.zip: 192MB [00:38, 4.71MB/s]\u001b[A\n",
      "ecpa.zip: 193MB [00:38, 4.09MB/s]\u001b[A\n",
      "ecpa.zip: 194MB [00:38, 4.92MB/s]\u001b[A\n",
      "ecpa.zip: 194MB [00:39, 5.32MB/s]\u001b[A\n",
      "ecpa.zip: 195MB [00:39, 4.86MB/s]\u001b[A\n",
      "ecpa.zip: 195MB [00:39, 5.09MB/s]\u001b[A\n",
      "ecpa.zip: 195MB [00:39, 4.45MB/s]\u001b[A\n",
      "ecpa.zip: 196MB [00:39, 4.17MB/s]\u001b[A\n",
      "ecpa.zip: 196MB [00:39, 3.56MB/s]\u001b[A\n",
      "ecpa.zip: 196MB [00:39, 3.99MB/s]\u001b[A\n",
      "ecpa.zip: 197MB [00:39, 4.49MB/s]\u001b[A\n",
      "ecpa.zip: 198MB [00:39, 4.74MB/s]\u001b[A\n",
      "ecpa.zip: 198MB [00:39, 4.85MB/s]\u001b[A\n",
      "ecpa.zip: 199MB [00:40, 5.18MB/s]\u001b[A\n",
      "ecpa.zip: 199MB [00:40, 5.56MB/s]\u001b[A\n",
      "ecpa.zip: 200MB [00:40, 5.49MB/s]\u001b[A\n",
      "ecpa.zip: 200MB [00:40, 4.72MB/s]\u001b[A\n",
      "ecpa.zip: 200MB [00:40, 3.57MB/s]\u001b[A\n",
      "ecpa.zip: 200MB [00:40, 3.24MB/s]\u001b[A\n",
      "ecpa.zip: 201MB [00:40, 3.54MB/s]\u001b[A\n",
      "ecpa.zip: 201MB [00:40, 3.75MB/s]\u001b[A\n",
      "ecpa.zip: 202MB [00:40, 4.46MB/s]\u001b[A\n",
      "ecpa.zip: 202MB [00:41, 4.28MB/s]\u001b[A\n",
      "ecpa.zip: 203MB [00:41, 4.37MB/s]\u001b[A\n",
      "ecpa.zip: 203MB [00:41, 4.03MB/s]\u001b[A\n",
      "ecpa.zip: 204MB [00:41, 4.68MB/s]\u001b[A\n",
      "ecpa.zip: 205MB [00:41, 5.16MB/s]\u001b[A\n",
      "ecpa.zip: 205MB [00:41, 4.66MB/s]\u001b[A\n",
      "ecpa.zip: 205MB [00:41, 3.75MB/s]\u001b[A\n",
      "ecpa.zip: 206MB [00:41, 3.67MB/s]\u001b[A\n",
      "ecpa.zip: 206MB [00:41, 4.27MB/s]\u001b[A\n",
      "ecpa.zip: 206MB [00:42, 4.08MB/s]\u001b[A\n",
      "ecpa.zip: 207MB [00:42, 3.26MB/s]\u001b[A\n",
      "ecpa.zip: 207MB [00:42, 2.70MB/s]\u001b[A\n",
      "ecpa.zip: 207MB [00:42, 2.46MB/s]\u001b[A\n",
      "ecpa.zip: 207MB [00:42, 2.83MB/s]\u001b[A\n",
      "ecpa.zip: 207MB [00:42, 2.32MB/s]\u001b[A\n",
      "ecpa.zip: 208MB [00:42, 2.35MB/s]\u001b[A\n",
      "ecpa.zip: 208MB [00:42, 2.08MB/s]\u001b[A\n",
      "ecpa.zip: 208MB [00:42, 2.69MB/s]\u001b[A\n",
      "ecpa.zip: 209MB [00:43, 3.30MB/s]\u001b[A\n",
      "ecpa.zip: 209MB [00:43, 3.44MB/s]\u001b[A\n",
      "ecpa.zip: 209MB [00:43, 3.05MB/s]\u001b[A\n",
      "ecpa.zip: 210MB [00:43, 2.93MB/s]\u001b[A\n",
      "ecpa.zip: 210MB [00:43, 2.88MB/s]\u001b[A\n",
      "ecpa.zip: 210MB [00:43, 2.60MB/s]\u001b[A\n",
      "ecpa.zip: 211MB [00:43, 3.75MB/s]\u001b[A\n",
      "ecpa.zip: 211MB [00:43, 3.30MB/s]\u001b[A\n",
      "ecpa.zip: 211MB [00:43, 2.47MB/s]\u001b[A\n",
      "ecpa.zip: 211MB [00:44, 2.26MB/s]\u001b[A\n",
      "ecpa.zip: 211MB [00:44, 2.23MB/s]\u001b[A\n",
      "ecpa.zip: 212MB [00:44, 2.29MB/s]\u001b[A\n",
      "ecpa.zip: 212MB [00:44, 2.20MB/s]\u001b[A\n",
      "ecpa.zip: 212MB [00:44, 1.90MB/s]\u001b[A\n",
      "ecpa.zip: 212MB [00:44, 1.90MB/s]\u001b[A\n",
      "ecpa.zip: 213MB [00:44, 2.47MB/s]\u001b[A\n",
      "ecpa.zip: 213MB [00:44, 3.29MB/s]\u001b[A\n",
      "ecpa.zip: 213MB [00:44, 2.71MB/s]\u001b[A\n",
      "ecpa.zip: 214MB [00:44, 2.79MB/s]\u001b[A\n",
      "ecpa.zip: 214MB [00:45, 2.78MB/s]\u001b[A\n",
      "ecpa.zip: 214MB [00:45, 3.26MB/s]\u001b[A\n",
      "ecpa.zip: 215MB [00:45, 2.89MB/s]\u001b[A\n",
      "ecpa.zip: 215MB [00:45, 3.20MB/s]\u001b[A\n",
      "ecpa.zip: 216MB [00:45, 3.61MB/s]\u001b[A\n",
      "ecpa.zip: 216MB [00:45, 4.02MB/s]\u001b[A\n",
      "ecpa.zip: 217MB [00:45, 4.14MB/s]\u001b[A\n",
      "ecpa.zip: 217MB [00:45, 3.46MB/s]\u001b[A\n",
      "ecpa.zip: 217MB [00:46, 3.57MB/s]\u001b[A\n",
      "ecpa.zip: 217MB [00:46, 2.84MB/s]\u001b[A\n",
      "ecpa.zip: 218MB [00:46, 2.50MB/s]\u001b[A\n",
      "ecpa.zip: 218MB [00:46, 1.90MB/s]\u001b[A\n",
      "ecpa.zip: 218MB [00:46, 2.24MB/s]\u001b[A\n",
      "ecpa.zip: 218MB [00:46, 2.49MB/s]\u001b[A\n",
      "ecpa.zip: 219MB [00:46, 2.55MB/s]\u001b[A\n",
      "ecpa.zip: 219MB [00:46, 1.93MB/s]\u001b[A\n",
      "ecpa.zip: 219MB [00:46, 1.86MB/s]\u001b[A\n",
      "ecpa.zip: 219MB [00:47, 1.59MB/s]\u001b[A\n",
      "ecpa.zip: 219MB [00:47, 1.73MB/s]\u001b[A\n",
      "ecpa.zip: 220MB [00:47, 2.40MB/s]\u001b[A\n",
      "ecpa.zip: 220MB [00:47, 2.18MB/s]\u001b[A\n",
      "ecpa.zip: 221MB [00:47, 4.64MB/s]\u001b[A\n",
      "ecpa.zip: 222MB [00:47, 4.70MB/s]\u001b[A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ecpa.zip: 222MB [00:47, 4.65MB/s]\u001b[A\n",
      "ecpa.zip: 223MB [00:47, 5.14MB/s]\u001b[A\n",
      "ecpa.zip: 223MB [00:48, 4.35MB/s]\u001b[A\n",
      "ecpa.zip: 223MB [00:48, 3.39MB/s]\u001b[A\n",
      "ecpa.zip: 224MB [00:48, 2.92MB/s]\u001b[A\n",
      "ecpa.zip: 225MB [00:48, 2.12MB/s]\u001b[A\n",
      "ecpa.zip: 227MB [00:49, 5.05MB/s]\u001b[A\n",
      "ecpa.zip: 229MB [00:49, 6.85MB/s]\u001b[A\n",
      "ecpa.zip: 229MB [00:49, 6.67MB/s]\u001b[A\n",
      "ecpa.zip: 230MB [00:49, 4.17MB/s]\u001b[A\n",
      "ecpa.zip: 231MB [00:49, 5.19MB/s]\u001b[A\n",
      "ecpa.zip: 232MB [00:49, 7.12MB/s]\u001b[A\n",
      "ecpa.zip: 233MB [00:49, 7.12MB/s]\u001b[A\n",
      "ecpa.zip: 234MB [00:50, 6.31MB/s]\u001b[A\n",
      "ecpa.zip: 234MB [00:50, 4.88MB/s]\u001b[A\n",
      "ecpa.zip: 234MB [00:50, 4.18MB/s]\u001b[A\n",
      "ecpa.zip: 234MB [00:50, 3.45MB/s]\u001b[A\n",
      "ecpa.zip: 234MB [00:50, 2.91MB/s]\u001b[A\n",
      "ecpa.zip: 234MB [00:50, 2.85MB/s]\u001b[A\n",
      "ecpa.zip: 235MB [00:50, 2.53MB/s]\u001b[A\n",
      "ecpa.zip: 235MB [00:50, 2.27MB/s]\u001b[A\n",
      "ecpa.zip: 235MB [00:50, 2.37MB/s]\u001b[A\n",
      "ecpa.zip: 235MB [00:51, 2.67MB/s]\u001b[A\n",
      "ecpa.zip: 235MB [00:51, 1.88MB/s]\u001b[A\n",
      "ecpa.zip: 236MB [00:51, 1.79MB/s]\u001b[A\n",
      "ecpa.zip: 236MB [00:51, 1.88MB/s]\u001b[A\n",
      "ecpa.zip: 236MB [00:51, 2.01MB/s]\u001b[A\n",
      "ecpa.zip: 236MB [00:51, 2.60MB/s]\u001b[A\n",
      "ecpa.zip: 237MB [00:51, 2.67MB/s]\u001b[A\n",
      "ecpa.zip: 237MB [00:51, 2.96MB/s]\u001b[A\n",
      "ecpa.zip: 238MB [00:51, 3.79MB/s]\u001b[A\n",
      "ecpa.zip: 238MB [00:52, 3.55MB/s]\u001b[A\n",
      "ecpa.zip: 238MB [00:52, 3.15MB/s]\u001b[A\n",
      "ecpa.zip: 238MB [00:52, 2.95MB/s]\u001b[A\n",
      "ecpa.zip: 239MB [00:52, 2.67MB/s]\u001b[A\n",
      "ecpa.zip: 239MB [00:52, 2.56MB/s]\u001b[A\n",
      "ecpa.zip: 239MB [00:52, 2.21MB/s]\u001b[A\n",
      "ecpa.zip: 239MB [00:52, 2.25MB/s]\u001b[A\n",
      "ecpa.zip: 239MB [00:52, 1.94MB/s]\u001b[A\n",
      "ecpa.zip: 240MB [00:52, 2.24MB/s]\u001b[A\n",
      "ecpa.zip: 240MB [00:53, 2.21MB/s]\u001b[A\n",
      "ecpa.zip: 240MB [00:53, 2.58MB/s]\u001b[A\n",
      "ecpa.zip: 241MB [00:53, 2.67MB/s]\u001b[A\n",
      "ecpa.zip: 241MB [00:53, 2.79MB/s]\u001b[A\n",
      "ecpa.zip: 241MB [00:53, 2.91MB/s]\u001b[A\n",
      "ecpa.zip: 242MB [00:53, 3.28MB/s]\u001b[A\n",
      "ecpa.zip: 242MB [00:53, 4.09MB/s]\u001b[A\n",
      "ecpa.zip: 242MB [00:53, 3.61MB/s]\u001b[A\n",
      "ecpa.zip: 243MB [00:53, 3.50MB/s]\u001b[A\n",
      "ecpa.zip: 243MB [00:53, 3.97MB/s]\u001b[A\n",
      "ecpa.zip: 244MB [00:54, 3.87MB/s]\u001b[A\n",
      "ecpa.zip: 244MB [00:54, 3.54MB/s]\u001b[A\n",
      "ecpa.zip: 244MB [00:54, 3.31MB/s]\u001b[A\n",
      "ecpa.zip: 245MB [00:54, 3.09MB/s]\u001b[A\n",
      "ecpa.zip: 245MB [00:54, 3.13MB/s]\u001b[A\n",
      "ecpa.zip: 246MB [00:54, 4.10MB/s]\u001b[A\n",
      "ecpa.zip: 246MB [00:54, 4.23MB/s]\u001b[A\n",
      "ecpa.zip: 246MB [00:54, 3.84MB/s]\u001b[A\n",
      "ecpa.zip: 246MB [00:54, 3.03MB/s]\u001b[A\n",
      "ecpa.zip: 247MB [00:55, 2.84MB/s]\u001b[A\n",
      "ecpa.zip: 247MB [00:55, 3.15MB/s]\u001b[A\n",
      "ecpa.zip: 247MB [00:55, 2.71MB/s]\u001b[A\n",
      "ecpa.zip: 247MB [00:55, 2.04MB/s]\u001b[A\n",
      "ecpa.zip: 248MB [00:55, 2.48MB/s]\u001b[A\n",
      "ecpa.zip: 248MB [00:55, 2.36MB/s]\u001b[A\n",
      "ecpa.zip: 248MB [00:55, 2.60MB/s]\u001b[A\n",
      "ecpa.zip: 248MB [00:55, 2.39MB/s]\u001b[A\n",
      "ecpa.zip: 249MB [00:55, 2.12MB/s]\u001b[A\n",
      "ecpa.zip: 249MB [00:56, 2.70MB/s]\u001b[A\n",
      "ecpa.zip: 249MB [00:56, 3.09MB/s]\u001b[A\n",
      "ecpa.zip: 250MB [00:56, 2.47MB/s]\u001b[A\n",
      "ecpa.zip: 250MB [00:56, 2.05MB/s]\u001b[A\n",
      "ecpa.zip: 250MB [00:56, 1.99MB/s]\u001b[A\n",
      "ecpa.zip: 250MB [00:56, 1.68MB/s]\u001b[A\n",
      "ecpa.zip: 250MB [00:56, 1.52MB/s]\u001b[A"
     ]
    }
   ],
   "source": [
    "generate_tei([\"adso100\"], \"tmp/\", ui_enabled=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "id": "925c59ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_tei(corpora_list, output_path, ui_enabled=True):\n",
    "    filename_list = []\n",
    "    for corpus in corpora_list:\n",
    "        if ui_enabled:\n",
    "            p = Path(output_path) / \"tmp_corp\" / corpus / \"averell\" / \"parser\"\n",
    "        else:\n",
    "            p = Path(output_path) / corpus / \"averell\" / \"parser\"\n",
    "        poem_path_list = p.glob(\"**/*.json\")\n",
    "        for poem_path in poem_path_list:\n",
    "            with open(poem_path, \"r\") as poem_file:\n",
    "                poem = json.load(poem_file)\n",
    "            file_name = poem_path.stem\n",
    "            filename_list.append(file_name)\n",
    "            poem_title = poem[\"poem_title\"]\n",
    "            author = poem[\"author\"]\n",
    "            corpus_name = CORPUS_NAMES[poem[\"corpus\"]]\n",
    "            manually_checked = poem[\"manually_checked\"]\n",
    "\n",
    "            poem_id = f\"{author}_{file_name}\"\n",
    "\n",
    "            root = et.Element(\"TEI\")\n",
    "            header = et.SubElement(root, \"teiHeader\")\n",
    "\n",
    "            file_desc = et.SubElement(header, \"fileDesc\")\n",
    "\n",
    "            title_stmt = et.SubElement(file_desc, \"titleStmt\")\n",
    "            title_stmt_desc = et.SubElement(title_stmt, \"title\")\n",
    "            author_stmt_desc = et.SubElement(title_stmt, \"author\")\n",
    "            title_stmt_desc.text = poem_title\n",
    "            author_stmt_desc.text = author\n",
    "\n",
    "            extent = et.SubElement(file_desc, \"extent\")\n",
    "\n",
    "            pub_stmt = et.SubElement(file_desc, \"publicationStmt\")\n",
    "            publisher = et.SubElement(pub_stmt, \"publisher\")\n",
    "            publisher.text = \"UNED University\"\n",
    "            idno = et.SubElement(pub_stmt, \"idno\")\n",
    "            idno.text = poem_id\n",
    "            availability = et.SubElement(pub_stmt, \"availability\")\n",
    "            availability.attrib[\"status\"] = \"free\"\n",
    "            p = et.SubElement(availability, \"p\")\n",
    "            p.text = \"The text is freely available.\"\n",
    "\n",
    "            series_stmt = et.SubElement(file_desc, \"seriesStmt\")\n",
    "            title_series = et.SubElement(series_stmt, \"title\")\n",
    "            title_series.text = corpus_name\n",
    "\n",
    "            source_desc = et.SubElement(file_desc, \"sourceDesc\")\n",
    "            bibl_source = et.SubElement(source_desc, \"bibl\")\n",
    "            bibl_title = et.SubElement(bibl_source, \"title\")\n",
    "            bibl_title.text = poem_title\n",
    "            bibl_author = et.SubElement(bibl_source, \"author\")\n",
    "            bibl_author.text = author\n",
    "\n",
    "            \n",
    "\n",
    "            measure_st = et.SubElement(extent, \"measure\")\n",
    "            measure_st.attrib[\"unit\"] = \"stanza\"\n",
    "            measure_st.text = str(len(poem[\"stanzas\"]))\n",
    "            n_lines = 0\n",
    "            \n",
    "            text_poem = et.SubElement(root, \"text\")\n",
    "            front_poem = et.SubElement(text_poem, \"front\")\n",
    "            head_poem = et.SubElement(front_poem, \"head\")\n",
    "            head_poem.text = poem_title\n",
    "            body_poem = et.SubElement(text_poem, \"body\")\n",
    "\n",
    "            lg_main = et.SubElement(body_poem, \"lg\")\n",
    "            lg_main.attrib[\"xmlns\"] = \"http://www.tei-c.org/ns/1.0\"\n",
    "            lg_main.attrib[\"type\"] = \"poem\"\n",
    "            \n",
    "            for stanza in poem[\"stanzas\"]:\n",
    "\n",
    "                n_lines += len(stanza[\"lines\"])\n",
    "\n",
    "                stanza_number = stanza[\"stanza_number\"]\n",
    "                stanza_type = stanza.get(\"stanza_type\")\n",
    "                lg = et.SubElement(lg_main, \"lg\")\n",
    "                lg.attrib[\"n\"] = str(stanza_number)\n",
    "                if stanza_type:\n",
    "                    lg.attrib[\"stanza_type\"] = stanza_type\n",
    "                for line in stanza[\"lines\"]:\n",
    "                    l = et.SubElement(lg, \"l\")\n",
    "                    l.text = line[\"line_text\"]\n",
    "                    l.attrib[\"n\"] = str(line[\"line_number\"])\n",
    "\n",
    "                    metrical_pattern = line.get(\"metrical_pattern\")\n",
    "                    rhyme = line.get(\"rhyme\")\n",
    "                    line_length = line.get(\"line_length\")\n",
    "\n",
    "                    if metrical_pattern:\n",
    "                        l.attrib[\"met\"] = str(metrical_pattern)\n",
    "                    if rhyme:\n",
    "                        l.attrib[\"rhyme\"] = str(rhyme)\n",
    "                    if line_length:\n",
    "                        l.attrib[\"line_length\"] = str(line_length)\n",
    "\n",
    "            measure_l = et.SubElement(extent, \"measure\")\n",
    "            measure_l.attrib[\"unit\"] = \"line\"\n",
    "            measure_l.text = str(n_lines)\n",
    "            tree = et.ElementTree(root)\n",
    "\n",
    "            # output_path = Path('corpora') / f'{poem[\"corpus\"]}' / 'averell' / 'TEI'\n",
    "            output_base_path = Path(output_path) / 'TEI'\n",
    "            output_extended_path = output_base_path / poem[\"corpus\"] / author\n",
    "\n",
    "            # prefix = '{:05d}'.format(filename_list.count(file_name))\n",
    "            output_file = f\"{poem_id}.xml\"\n",
    "            if not os.path.exists(output_base_path):\n",
    "                print(\"no esite\")\n",
    "                Path.mkdir(output_base_path)\n",
    "            if not os.path.exists(output_extended_path):\n",
    "                Path.mkdir(output_extended_path, parents=True)\n",
    "            et.indent(tree, space=\" \", level=0)\n",
    "            tree.write(f\"{Path(output_extended_path) / output_file}\", encoding=\"UTF-8\",\n",
    "                       xml_declaration=True)\n",
    "    if ui_enabled:\n",
    "        with ZipFile(\"tmp/TEI.zip\", \"w\") as zfile:\n",
    "            for corpus in corpora_list:\n",
    "                print(corpus)\n",
    "\n",
    "                p = Path(f'tmp/TEI/{corpus}')\n",
    "                for f in p.glob(\"*/**/*.xml\"):\n",
    "                    zfile.write(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "id": "66a317a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "available_languages = get_available_languages(CORPORA_SOURCES)\n",
    "PARAMS = {\n",
    "    \"granularity\": \"poem\",\n",
    "    \"output_format\": \"TEI\",\n",
    "    \"corpora_list\": [\n",
    "        'bibit', 'stichopt', 'disco2_1',\n",
    "         'disco3', 'adso', 'adso100',\n",
    "         'plc', 'gongo', 'ecpa',\n",
    "         '4b4v', 'czverse', 'mel'\n",
    "    ],\n",
    "}\n",
    "\n",
    "with gr.Blocks() as app_averell:\n",
    "    def export_corpora(output_path, output_format, output_granularity):\n",
    "        corpora_list = PARAMS[\"corpora_list\"]\n",
    "        if not corpora_list:\n",
    "            return {out_file: gr.File.update(label=\"ERROR: No corpus selected\")}\n",
    "        output_path = output_path[\"label\"]\n",
    "        print(output_path)\n",
    "        filename = f\"tmp/{output_format}.zip\"\n",
    "        # Export to TEI\n",
    "        if output_format == \"TEI\":\n",
    "            print(\"TEI\")\n",
    "            generate_tei(corpora_list, output_path, ui_enabled=True)\n",
    "            return {out_file: gr.File.update(value=f\"tmp/{output_format}.zip\",\n",
    "                                             label=output_format)}\n",
    "        # Expor to JSON\n",
    "        if output_granularity == \"poem\":\n",
    "            with ZipFile(filename, \"w\") as zfile:\n",
    "                for corpus in corpora_list:\n",
    "                    p = Path(f'{output_path}/{output_format}/{corpus}')\n",
    "                    for f in p.glob(\"**/*.json\"):\n",
    "                        zfile.write(f)\n",
    "            return {out_file: gr.File.update(value=f\"tmp/{output_format}.zip\",\n",
    "                                             label=output_format)}\n",
    "        else:\n",
    "            json_l, filename = export_corpora(get_ids(corpora_list),\n",
    "                                                 output_granularity,\n",
    "                                                 \"tmp/tmp_corp\",\n",
    "                                                 None,\n",
    "                                                 False)\n",
    "            with open(f\"tmp/{filename}.json\", 'w', encoding='utf-8') as f:\n",
    "                json.dump(json_l, f, ensure_ascii=False, indent=4)\n",
    "            return {out_file: gr.File.update(value=f\"tmp/{filename}.json\",\n",
    "                                             label=filename)}\n",
    "\n",
    "\n",
    "    def block_granularity(rad_format):\n",
    "        if rad_format == \"TEI\":\n",
    "            PARAMS[\"output_format\"] = rad_format\n",
    "            return {rad_granularity: gr.update(value=\"poem\", visible=False)}\n",
    "        else:\n",
    "            PARAMS[\"output_format\"] = rad_format\n",
    "            return {rad_granularity: gr.update(value=\"poem\", visible=True)}\n",
    "    def update_granularity(value):\n",
    "        PARAMS[\"granularity\"] = value\n",
    "        true_l = [gr.Checkbox.update(value=True, visible=True)]\n",
    "        false_l = [gr.Checkbox.update(value=False, visible=False)] # False\n",
    "        true_acc = [gr.Box.update(visible=True)]\n",
    "        false_acc = [gr.Box.update(visible=False)] # False\n",
    "        if value == \"word\":\n",
    "            # 000011100110 + 110110 + 110110\n",
    "            corp_list = false_l*4 + true_l*3 + false_l*2 + true_l*2 + false_l\n",
    "            lang_list = true_l*2 + false_l + true_l*2 + false_l\n",
    "            acc_list = true_acc*2 + false_acc + true_acc*2 + false_acc\n",
    "            PARAMS[\"corpora_list\"] = [\"plc\", \"gongo\", \"ecpa\", \"bibit\", \"czverse\"]\n",
    "            return corp_list + lang_list + acc_list\n",
    "        elif value == \"syllable\":\n",
    "            # 000011000000 + 100000 + 100000\n",
    "            corp_list = false_l*4 + true_l*2 + false_l*6\n",
    "            lang_list = true_l + false_l*5\n",
    "            acc_list = true_acc + false_acc*5\n",
    "            PARAMS[\"corpora_list\"] = [\"plc\", \"gongo\"]\n",
    "            return corp_list + lang_list + acc_list\n",
    "        else:\n",
    "            return true_l*18 + true_acc*6\n",
    "\n",
    "\n",
    "    def change_selection(value, *labels):\n",
    "        for corpus in labels:\n",
    "            update_corpora_list(value, corpus)\n",
    "        return value\n",
    "    def change_global_selection(value, *labels):\n",
    "        for corpus in labels:\n",
    "            update_corpora_list(value, corpus)\n",
    "        return [gr.Checkbox.update(value=value)] * 6\n",
    "    def update_corpora_list(added, corpus):\n",
    "        corpora_list = PARAMS[\"corpora_list\"]\n",
    "        if corpus in corpora_list and not added:\n",
    "            corpora_list.remove(corpus)\n",
    "        elif corpus not in corpora_list and added:\n",
    "            corpora_list.append(corpus)\n",
    "\n",
    "    app_title = gr.HTML(\"<h1>Averell</h1>\")\n",
    "    with gr.Row() as row:\n",
    "        with gr.Column(scale=3) as c1:\n",
    "            rad_format = gr.Radio([\"TEI\", \"JSON\"],\n",
    "                                  label=\"Output\",\n",
    "                                  info=\"Choose output format\",\n",
    "                                  value=\"TEI\",\n",
    "                                  interactive=True)\n",
    "            rad_granularity = gr.Radio(\n",
    "                [\"poem\", \"stanza\", \"line\", \"word\", \"syllable\"],\n",
    "                label=\"Granularity\",\n",
    "                info=\"Choose output granularity\",\n",
    "                value=\"poem\",\n",
    "                interactive=True,\n",
    "                visible=False,\n",
    "                )\n",
    "            corpus_checkboxes = []\n",
    "            lang_checkboxes = []\n",
    "            with gr.Box() as b1:\n",
    "                gr.HTML(value=\"<h3>Corpora list<h3>\")\n",
    "                gr.HTML(value=\"<br>\")\n",
    "                all_corp_chk = gr.Checkbox(True, label=\"Select all/none\",\n",
    "                                           interactive=True)\n",
    "                with gr.Row() as rowa:\n",
    "                    all_label_list = []\n",
    "                    gr.Checkbox(label=\"dummy\",visible=False)\n",
    "                    for lang in available_languages:\n",
    "                        with gr.Column() as b2:\n",
    "                            language = pycountry.languages.get(\n",
    "                                alpha_2=lang).name\n",
    "                            gr.HTML(language)\n",
    "                            with gr.Blocks() as corpora:\n",
    "                                with gr.Row() as rowb:\n",
    "                                    lang_chk = gr.Checkbox(True,\n",
    "                                                           label=\"Select all/none\",\n",
    "                                                           interactive=True)\n",
    "                                    filtered_corpus = filter_corpus_language(\n",
    "                                        CORPORA_SOURCES, lang)\n",
    "                                    with gr.Accordion(\"Expand list\",\n",
    "                                                      open=False) as acc:\n",
    "                                        label_list = []\n",
    "                                        for corpus in filtered_corpus:\n",
    "                                            classes = corpus[\"properties\"][\n",
    "                                                \"granularity\"]\n",
    "                                            classes.append(\"poem\")\n",
    "                                            classes.append(lang)\n",
    "                                            chk = gr.Checkbox(True,\n",
    "                                                              label=corpus[\n",
    "                                                                  \"name\"],\n",
    "                                                              info=f'License: {corpus[\"properties\"][\"license\"]} | Number of poems: {corpus[\"properties\"][\"doc_quantity\"]}',\n",
    "                                                              interactive=True,\n",
    "                                                              elem_classes=classes,\n",
    "                                                              elem_id=corpus[\n",
    "                                                                  \"properties\"][\n",
    "                                                                  \"slug\"],\n",
    "                                                              )\n",
    "                                            label = gr.Textbox(\n",
    "                                                value=corpus[\"properties\"][\n",
    "                                                    \"slug\"], visible=False)\n",
    "                                            # Corpus checkboxes change\n",
    "                                            chk.change(update_corpora_list,\n",
    "                                                       [chk, label],\n",
    "                                                       show_progress=False)\n",
    "                                            # \"Select all\" language checkboxes change\n",
    "                                            lang_chk.change(change_selection,\n",
    "                                                            [lang_chk,\n",
    "                                                             *label_list],\n",
    "                                                            chk,\n",
    "                                                            show_progress=False)\n",
    "                                            corpus_checkboxes.append(chk)\n",
    "                                            label_list.append(label)\n",
    "                                            all_label_list.append(label)\n",
    "                                        lang_checkboxes.append(lang_chk)\n",
    "                # \"Select All/None\" checkbox change\n",
    "                all_corp_chk.change(change_global_selection,\n",
    "                                    [all_corp_chk,\n",
    "                                     *all_label_list],\n",
    "                                    [*lang_checkboxes])\n",
    "        with gr.Column(scale=1) as c2:\n",
    "            accordions_boxes = rowa.children[1:]\n",
    "            rad_granularity.change(update_granularity,\n",
    "                                   rad_granularity,\n",
    "                                   [*corpus_checkboxes,\n",
    "                                    *lang_checkboxes,\n",
    "                                    *accordions_boxes],\n",
    "                                   show_progress=False)\n",
    "            rad_format.change(block_granularity,\n",
    "                              rad_format,\n",
    "                              rad_granularity,\n",
    "                              api_name=\"output_format\",\n",
    "                              show_progress=False)\n",
    "            exp_btn = gr.Button(\"Export\")\n",
    "            folder_path = gr.Label(value=\"tmp/\", visible=False)\n",
    "            out_file = gr.File()\n",
    "            exp_btn.click(export_corpora,\n",
    "                          [folder_path,\n",
    "                           rad_format,\n",
    "                           rad_granularity],\n",
    "                          out_file,\n",
    "                          api_name=\"export\")\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "id": "db7a5494",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running on local URL:  http://127.0.0.1:7874\n",
      "\n",
      "To create a public link, set `share=True` in `launch()`.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><iframe src=\"http://127.0.0.1:7874/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 212,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp/\n",
      "tmp/\n",
      "tmp/\n",
      "tmp/\n",
      "tmp/\n",
      "tmp/\n",
      "tmp/\n",
      "tmp/\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                   | 0/1 [00:00<?, ?it/s]\n",
      "disco3.zip: 0.00B [00:00, ?B/s]\u001b[A\n",
      "disco3.zip: 8.00kB [00:02, 2.97kB/s]\u001b[A\n",
      "disco3.zip: 184kB [00:02, 91.3kB/s] \u001b[A\n",
      "disco3.zip: 1.48MB [00:02, 981kB/s]\u001b[A\n",
      "disco3.zip: 2.43MB [00:03, 1.74MB/s]\u001b[A\n",
      "disco3.zip: 3.39MB [00:03, 2.65MB/s]\u001b[A\n",
      "disco3.zip: 4.41MB [00:03, 3.73MB/s]\u001b[A\n",
      "disco3.zip: 5.29MB [00:03, 4.61MB/s]\u001b[A\n",
      "disco3.zip: 6.20MB [00:03, 5.51MB/s]\u001b[A\n",
      "disco3.zip: 6.99MB [00:03, 6.11MB/s]\u001b[A\n",
      "disco3.zip: 7.81MB [00:03, 6.68MB/s]\u001b[A\n",
      "disco3.zip: 8.61MB [00:03, 6.94MB/s]\u001b[A\n",
      "disco3.zip: 9.52MB [00:03, 7.60MB/s]\u001b[A\n",
      "disco3.zip: 10.5MB [00:03, 8.37MB/s]\u001b[A\n",
      "disco3.zip: 11.5MB [00:04, 8.93MB/s]\u001b[A\n",
      "disco3.zip: 12.5MB [00:04, 9.26MB/s]\u001b[A\n",
      "disco3.zip: 13.5MB [00:04, 9.57MB/s]\u001b[A\n",
      "disco3.zip: 14.5MB [00:04, 9.90MB/s]\u001b[A\n",
      "disco3.zip: 15.5MB [00:04, 10.2MB/s]\u001b[A\n",
      "disco3.zip: 16.5MB [00:04, 10.1MB/s]\u001b[A\n",
      "disco3.zip: 17.4MB [00:04, 9.86MB/s]\u001b[A\n",
      "disco3.zip: 18.2MB [00:04, 9.53MB/s]\u001b[A\n",
      "disco3.zip: 19.1MB [00:04, 9.40MB/s]\u001b[A\n",
      "disco3.zip: 20.0MB [00:04, 9.31MB/s]\u001b[A\n",
      "disco3.zip: 20.8MB [00:05, 9.21MB/s]\u001b[A\n",
      "disco3.zip: 21.7MB [00:05, 9.10MB/s]\u001b[A\n",
      "disco3.zip: 22.5MB [00:05, 8.94MB/s]\u001b[A\n",
      "disco3.zip: 23.2MB [00:05, 8.47MB/s]\u001b[A\n",
      "disco3.zip: 24.0MB [00:05, 8.32MB/s]\u001b[A\n",
      "disco3.zip: 24.7MB [00:05, 7.73MB/s]\u001b[A\n",
      "disco3.zip: 25.4MB [00:05, 7.60MB/s]\u001b[A\n",
      "disco3.zip: 26.1MB [00:05, 7.72MB/s]\u001b[A\n",
      "disco3.zip: 26.5MB [00:05, 5.24MB/s]\u001b[A\n",
      "disco3.zip: 29.7MB [00:06, 5.08MB/s]\u001b[A\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 1/1 [00:06<00:00,  6.79s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp/\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                   | 0/1 [00:00<?, ?it/s]\n",
      "adso100.zip: 0.00B [00:00, ?B/s]\u001b[A\n",
      "adso100.zip:   6%|███▌                                                    | 8.00k/128k [00:04<01:10, 1.74kB/s]\u001b[A\n",
      "adso100.zip: 128kB [00:04, 27.2kB/s]                                                                          \u001b[A\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 1/1 [00:04<00:00,  4.82s/it]\n",
      "INFO:root:Downloaded ADSO 100 poems corpus corpus\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp/\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                   | 0/1 [00:00<?, ?it/s]\n",
      "disco2_1.zip: 0.00B [00:00, ?B/s]\u001b[A\n",
      "disco2_1.zip: 8.00kB [00:04, 1.71kB/s]\u001b[A\n",
      "disco2_1.zip: 416kB [00:04, 122kB/s]  \u001b[A\n",
      "disco2_1.zip: 1.88MB [00:05, 730kB/s]\u001b[A\n",
      "disco2_1.zip: 2.84MB [00:05, 1.22MB/s]\u001b[A\n",
      "disco2_1.zip: 3.84MB [00:05, 1.88MB/s]\u001b[A\n",
      "disco2_1.zip: 4.80MB [00:05, 2.64MB/s]\u001b[A\n",
      "disco2_1.zip: 5.74MB [00:05, 3.47MB/s]\u001b[A\n",
      "disco2_1.zip: 6.55MB [00:05, 4.17MB/s]\u001b[A\n",
      "disco2_1.zip: 7.41MB [00:05, 4.96MB/s]\u001b[A\n",
      "disco2_1.zip: 8.25MB [00:05, 5.71MB/s]\u001b[A\n",
      "disco2_1.zip: 9.19MB [00:05, 6.60MB/s]\u001b[A\n",
      "disco2_1.zip: 10.1MB [00:05, 7.41MB/s]\u001b[A\n",
      "disco2_1.zip: 11.1MB [00:06, 8.05MB/s]\u001b[A\n",
      "disco2_1.zip: 12.1MB [00:06, 8.61MB/s]\u001b[A\n",
      "disco2_1.zip: 13.1MB [00:06, 9.09MB/s]\u001b[A\n",
      "disco2_1.zip: 13.9MB [00:06, 9.06MB/s]\u001b[A\n",
      "disco2_1.zip: 15.1MB [00:06, 9.83MB/s]\u001b[A\n",
      "disco2_1.zip: 16.0MB [00:06, 9.90MB/s]\u001b[A\n",
      "disco2_1.zip: 16.9MB [00:06, 9.73MB/s]\u001b[A\n",
      "disco2_1.zip: 17.8MB [00:06, 9.52MB/s]\u001b[A\n",
      "disco2_1.zip: 18.6MB [00:06, 9.29MB/s]\u001b[A\n",
      "disco2_1.zip: 19.5MB [00:06, 9.37MB/s]\u001b[A\n",
      "disco2_1.zip: 21.9MB [00:07, 3.26MB/s]\u001b[A\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 1/1 [00:07<00:00,  7.51s/it]\n",
      "INFO:root:Downloaded Disco V2.1 corpus\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp/\n",
      "tmp/\n",
      "TEI\n"
     ]
    }
   ],
   "source": [
    "app_averell.launch(share=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76e24030",
   "metadata": {},
   "outputs": [],
   "source": [
    "app_averell.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "id": "7a40f841",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'granularity': 'word',\n",
       " 'output_format': 'JSON',\n",
       " 'corpora_list': ['plc', 'gongo', 'ecpa', 'bibit', 'czverse']}"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PARAMS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "90b8ef0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "corp_list = [{'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': False, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': False, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': False, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': False, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': True, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': True, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': False, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': False, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': False, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': False, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': False, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': False, '__type__': 'update'}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "12757fa9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " True,\n",
       " True,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False]"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[c[\"value\"] for c in corp_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "e24c8068",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['disco2_1',\n",
       " 'disco3',\n",
       " 'adso',\n",
       " 'adso100',\n",
       " 'plc',\n",
       " 'gongo',\n",
       " 'ecpa',\n",
       " '4b4v',\n",
       " 'mel',\n",
       " 'bibit',\n",
       " 'czverse',\n",
       " 'stichopt']"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "else\n",
      "syllable\n",
      "[{'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': False, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': False, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': False, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': False, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': True, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': True, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': False, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': False, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': False, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': False, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': False, '__type__': 'update'}, {'label': None, 'show_label': None, 'interactive': None, 'visible': True, 'value': False, '__type__': 'update'}]\n"
     ]
    }
   ],
   "source": [
    "PARAMS[\"corpora_list\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1230f4a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "CORPUS_NAMES = {\n",
    "    \"disco2_1\": \"Disco V2.1\",\n",
    "    \"disco3\": \"Disco V3\",\n",
    "    \"adso\": \"Sonetos Siglo de Oro\",\n",
    "    \"adso100\": \"ADSO 100 poems corpus\",\n",
    "    \"plc\": \"Poesía Lírica Castellana Siglo de Oro\",\n",
    "    \"gongo\": \"Gongocorpus\",\n",
    "    \"ecpa\": \"Eighteenth Century Poetry Archive\",\n",
    "    \"4b4v\": \"For Better For Verse\",\n",
    "    \"mel\": \"Métrique en Ligne\",\n",
    "    \"bibit\": \"Biblioteca Italiana\",\n",
    "    \"czverse\": \"Corpus of Czech Verse\",\n",
    "    \"stichopt\": \"Stichotheque Portuguese\",\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6cf96167",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3765e435",
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_tei([\"adso100\"], \"tmp\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87e6a189",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}