config.py

# OL DUMPS SOURCE 👉 https://openlibrary.org/developers/dumps 👈

# HDFS
hadoop = {
    "local_path_file": "C:/Users/39346/dati_progetto_dm",
    "hdfs_path_file": "/data_dm_project/",
}

# SCRAPING FELTRINELLI
scraping_fltr = {
    "books_for_page": 500,
    "timeout": 240,
    "path_output": "./data/feltrinelli",
    "name_file_out": "FELTRINELLI_ENGLISH_BOOKS.csv",
}

# PREPROCESSING FELTRINELLI
preprocessing_fltr = {
    "input_file_path": "./data/feltrinelli/FELTRINELLI_ENGLISH_BOOKS.csv",
    "path_file_out": "./data/feltrinelli/cleaned",
    "name_file_out": "PREPROCESSED_FELTRINELLI_ENGLISH_BOOKS.csv",
}

# SCRAPING NY
scraping_ny = {
    "starting_index": 1,
    "ending_index": 31951,
    "timeout": 60,
    "out_dir_split": "./data/new_york/splitted_csv",
    "out_dir_full_csv": "./data/new_york/full_catalog",
    "name_full_csv": "00_ny_pub_lib.csv",
}

# SCRAPING LONDON
scraping_lnd = {
    "starting_index": 1,
    "ending_index": 31951,
    "timeout": 60,
    "out_dir_split": "./data/london/splitted_csv",
    "out_dir_full_csv": "./data/london/full_catalog",
    "name_full_csv": "00_raw_london_full_catalog.csv",
}


# OL - TXT TO BOOKS
books = {
    # libri_completi.txt = "editions dump" on Open Library
    "path_input_books": "./data/openlibrary/SITO_OPENLIBRARY/libri_completi.txt",
    "path_splitted_catalog": "./data/openlibrary/RAW_CSV/books/SPLITTED",
    "path_full_catalog": "./data/openlibrary/RAW_CSV/books/FULL/",
    "name_full_books_catalog": "00_RAW_books_V3.csv",
    "books_cols": [
        "key",
        "title",
        "publishers",
        "languages",
        "number_of_pages",
        "physical_format",
        "isbn_13",
        "publish_date",
        "authors",
        "subjects",
    ],
    "dtypes": {
        "key": "str",
        "title": "str",
        "publishers": "str",
        "languages": "str",
        "number_of_pages": "float",
        "physical_format": "str",
        "isbn_13": "str",
        "publish_date": "str",
        "authors": "str",
        "subjects": "str",
    },
    "books_chunksize": 200000,
}

# OL - TXT TO AUTHORS
authors = {
    # autori_piccolo.txt = "authors dump" on Open Library
    "path_input_authors": "./data/openlibrary/SITO_OPENLIBRARY/autori_piccolo.txt",
    "path_splitted_catalog": "./data/openlibrary/RAW_CSV/authors/SPLITTED",
    "path_full_catalog": "./data/openlibrary/RAW_CSV/authors/FULL/",
    "name_full_authors_catalog": "00_RAW_authors_V3.csv",
    "authors_cols": ["key", "name"],
    "dtypes": {"key": "str", "name": "str"},
    "authors_chunksize": 200000,
}

# OL - PREPROCESS RAW AUTHORS
prp_raw_authors = {
    "raw_ol_authors_path": "./data/openlibrary/RAW_CSV/authors/FULL/00_RAW_authors.csv",
    "dtypes_auth": {"key": "str", "name": "str"},
}

# OL - PREPROCESS RAW BOOKS
prp_raw_books = {
    "raw_ol_books_path": "./data/openlibrary/RAW_CSV/books/FULL/00_RAW_books.csv",
    "dtypes_books": {
        "key": "str",
        "title": "str",
        "publishers": "str",
        "languages": "str",
        "number_of_pages": "str",
        "physical_format": "str",
        "isbn_13": "str",
        "publish_date": "str",
        "authors": "str",
        "subjects": "str",
    },
}

# OL - JOIN BOOKS AND AUTHORS
ol_join = {
    "output_file_out": "./data/openlibrary/RAW_CSV/joined_auth_books",
    "name_csv_out": "raw_joined_authors_books.csv",
    "join_mode": "left",
    "join_key": "OL_key_author",
}

########### ENRICHMENT
enrich_fltr = {
    "fltr_cleaned_path": "./data/feltrinelli/cleaned/PREPROCESSED_FELTRINELLI_ENGLISH_BOOKS.csv",
    # sostituire con 'data\openlibrary\RAW_CSV\joined_auth_books\raw_joined_authors_books.csv', lft_jn_ol.csv è solo un campione
    "ol_joined_auth_book_path": "./data/feltrinelli/enrich_OL/lft_jn_ol.csv",
    "fltr_left_enr_path_out": "./data/feltrinelli/enrich_OL/",
    "fltr_left_enr_name_file_out": "left_j_fltr_OL.csv",
}

enrich_mondadori = {
    "path_output": "./data/feltrinelli/enrich_Mondadori",
    "name_file_out": "MONDADORI_ENRICH.csv",
}

enrich_hoepli = {
    "path_output": "./data/feltrinelli/enrich_Hoepli",
    "name_file_out": "final_hoepli_enrich.csv",
    "timeout": 15,
}

######## DATASET ANALYSIS

data_analysis = {
    "data_set_path": "./data/feltrinelli/redefine/enrch_hoepli.csv",
    "corrected_year_path": "./data/feltrinelli/redefine/def_anno.xlsx",
    "corrected_category_path": "./data/feltrinelli/redefine/categorie.xlsx",
}