Skip to content

Commit

Permalink
Merge branch 'release/v0.3'
Browse files Browse the repository at this point in the history
  • Loading branch information
Nahue committed Jun 20, 2019
2 parents f0d1180 + 32ccfa6 commit aeda076
Show file tree
Hide file tree
Showing 4 changed files with 345 additions and 124 deletions.
8 changes: 5 additions & 3 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,15 @@
"scripts": {
"eslint": "eslint --ext .js --fix --ignore-path .gitignore .",
"predev": "npm run eslint",
"watch": "nodemon --inspect-brk=5858 --exec babel-node src/index.js --watch src",
"dev": "babel-node src/index.js --inspect-brk=5858",
"build": "babel src --out-dir dist",
"start": "NODE_ENV=production node dist/index.js",
"csv-votaciones": "for file in ./$npm_package_config_folder/*.json; do json2csv -i $file >> ./$npm_package_config_folder/votaciones.csv; done",
"csv-votaciones": "for file in ./$npm_package_config_folder/*.json; do json2csv -i $file --fields-config=$npm_package_config_fields_config >> ./$npm_package_config_folder/votaciones.csv; done",
"csv-votos": "for file in ./$npm_package_config_folder/votos/$npm_package_config_year/*.json; do json2csv -i $file >> ./$npm_package_config_folder/votaciones_votos-$npm_package_config_year.csv; done",
"diputados-csv-votaciones": "npm run csv-votaciones --app:folder=data/diputados",
"csv-expedientes": "for file in ./$npm_package_config_folder/expedientes/*.json; do json2csv -i $file >> ./$npm_package_config_folder/votaciones_expedientes.csv; done",
"diputados-csv-votaciones": "npm run csv-votaciones --app:folder=data/diputados --app:fields_config=src/providers/ar-diputados/votingsCsv.json",
"diputados-csv-expedientes": "npm run csv-expedientes --app:folder=data/diputados",
"diputados-csv-votos": "npm run csv-votos --app:folder=data/diputados",
"senadores-csv-votaciones": "npm run csv-votaciones --app:folder=data/senadores",
"senadores-csv-votos": "npm run csv-votos --app:folder=data/senadores"
},
Expand Down
120 changes: 72 additions & 48 deletions src/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,25 +8,33 @@ const defaultYear = now.getFullYear();

yargs
.command({
command: "votaciones <provider> <anio>",
command: "votaciones <provider> <anio> [anioMax]",
desc: "Descarga el listado de votaciones de <provider> del <anio> indicado",
builder: yargs => yargs.default("anio", defaultYear),
handler: argv => parseVotingsFromYear(argv.provider, argv.anio)
handler: argv =>
parseVotingsFromYear(argv.provider, argv.anio, argv.anioMax)
})
.command({
command: "votos <provider> <anio>",
command: "votos <provider> <anio> [anioMax] [soloEstasVotaciones..]",
desc:
"Descarga los votos cada votación de <provider> realizada durante el <anio> indicado",
builder: yargs => yargs.default("anio", defaultYear),
handler: argv => parseVotingsDetailsFromYear(argv.provider, argv.anio)
handler: argv =>
parseVotingsDetailsFromYear(
argv.provider,
argv.anio,
argv.anioMax,
argv.soloEstasVotaciones
)
})
.command({
command: "importar <provider> <anio> [soloEstasVotaciones..]",
command: "importar <provider> <anio> [anioMax] [soloEstasVotaciones..]",
desc: "Importa todo lo descargado de <provider> para el <anio> en el API",
builder: yargs => yargs.default("anio", defaultYear),
handler: argv =>
getProvider(argv.provider).api.sendYear(
argv.anio,
argv.anioMax,
argv.soloEstasVotaciones
)
})
Expand All @@ -53,66 +61,82 @@ function getProvider(providerType) {
return provider.default;
}

async function parseVotingsFromYear(providerType, year) {
async function parseVotingsFromYear(providerType, yearMin, yearMax = null) {
if (!yearMax) {
yearMax = yearMin;
}
const provider = getProvider(providerType);
const scrapper = new provider.scrapper();
try {
logger.info("INICIO DEL ANALISIS DEL AÑO", year);
await scrapper.start();
for (let year = yearMin; year <= yearMax; year++) {
try {
const votings = await scrapper.parseVotingsFromYear(year);
if (votings.length) {
const path = await persistData(providerType, `${year}.json`, votings);
logger.info(`Votaciones guardadas. Archivo: ${path}`);
logger.info("INICIO DEL ANALISIS DEL AÑO", year);
await scrapper.start();
try {
await scrapper.parseVotingsFromYear(year);
} catch (error) {
logger.error(`parseVotingsFromYear: ${error.message}`);
}
} catch (error) {
logger.error(`parseVotingsFromYear: ${error.message}`);
} catch (err) {
logger.error(`Ocurrió un error general durante el proceso ${err}`);
} finally {
await scrapper.finish();
logger.info(`FIN DEL ANALISIS DEL AÑO ${year}`);
}
} catch (err) {
logger.error(`Ocurrió un error general durante el proceso ${err}`);
} finally {
await scrapper.finish();
logger.info(`FIN DEL ANALISIS DEL AÑO ${year}`);
process.exit();
}
process.exit();
}

async function parseVotingsDetailsFromYear(providerType, year) {
async function parseVotingsDetailsFromYear(
providerType,
yearMin,
yearMax = null,
onlyTheseVotings = []
) {
if (!yearMax) {
yearMax = yearMin;
}
const provider = getProvider(providerType);
const scrapper = new provider.scrapper();
try {
logger.info(`INICIO ANALISIS DE VOTACIONES DEL AÑO ${year}`);
for (let year = yearMin; year <= yearMax; year++) {
try {
await scrapper.start();
const database = getDataFromFile(`${providerType}/${year}.json`);
const page = await scrapper.createPage();
const editedVotings = [];
for (let voting of database) {
const editedVoting = await scrapper.parseVotingsDetails(
page,
voting,
`${providerType}/votos/${year}`
);
logger.info(`INICIO ANALISIS DE VOTACIONES DEL AÑO ${year}`);
try {
await scrapper.start();
const database = getDataFromFile(`${providerType}/${year}.json`);
const page = await scrapper.createPage();
const editedVotings = [];
for (let voting of database) {
let editedVoting = voting;
if (
!onlyTheseVotings.length ||
onlyTheseVotings.indexOf(voting.id) > -1
) {
editedVoting = await scrapper.parseVotingsDetails(
page,
voting,
`${providerType}/votos/${year}`
);
}
editedVotings.push(editedVoting);
}

editedVotings.push(editedVoting);
const path = await persistData(
providerType,
`${year}.json`,
editedVotings
);
logger.info(`Votaciones actualizadas. Archivo: ${path}`);
} catch (err) {
logger.error(err.stack);
}

const path = await persistData(
providerType,
`${year}.json`,
editedVotings
);
logger.info(`Votaciones actualizadas. Archivo: ${path}`);
} catch (err) {
logger.error(err.stack);
} finally {
await scrapper.finish();
logger.info(`FIN ANALISIS DE VOTACIONES DEL AÑO: ${year}`);
}
} catch (err) {
logger.error(err.stack);
} finally {
await scrapper.finish();
logger.info(`FIN ANALISIS DE VOTACIONES DEL AÑO: ${year}`);
process.exit();
}
process.exit();
}

// async function fillVotingsDetailsFromYear(providerType, year) {
Expand Down
Loading

0 comments on commit aeda076

Please sign in to comment.