Skip to content
This repository has been archived by the owner on Dec 13, 2022. It is now read-only.

fix import find gap querying limit #33

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix import find gap querying limit #33

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 23 additions & 15 deletions src/database/doctor.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,25 +48,33 @@ export const findBlockGaps = async (

const missingHeights: number[] = [];
for (const [index, heightGroup] of queryHeightGroups.entries()) {
const blockQ: { rows: unknown[] } = await cassandraClient.execute(
`SELECT block_height FROM ${KEYSPACE}.block_height_sorted_asc WHERE nth_million = ${index} AND block_height >= ${R.head(
heightGroup
)} AND block_height <= ${R.last(heightGroup)}`
const startHeight = R.head(heightGroup);
const endHeight = R.last(heightGroup);
const queryHeightSubGroups = R.splitEvery(
5000, // TODO: cassandra-driver has a limit of 5000, should refactor to `eachRow()`
R.range(startHeight, endHeight + 1)
);
for (const height of R.range(
R.head(heightGroup),
R.last(heightGroup) + 1
)) {
const findResult = R.findIndex(
(row: { block_height: CassandraTypes.Long }) =>
row.block_height.equals(height)
)(blockQ.rows as { block_height: CassandraTypes.Long }[]);
if (findResult < 0) {
missingHeights.push(height);
for (const subHeightGroup of queryHeightSubGroups) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hats off for using Ramda functions!

could we use for-await here and have the pagination be done automatically, just thinking.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure will come back to this when I have some spare time...

const blockQ: { rows: unknown[] } = await cassandraClient.execute(
`SELECT block_height FROM ${KEYSPACE}.block_height_sorted_asc WHERE nth_million = ${index} AND block_height >= ${R.head(
subHeightGroup
)} AND block_height <= ${R.last(subHeightGroup)}`
);
for (const height of R.range(
R.head(subHeightGroup),
R.last(subHeightGroup) + 1
)) {
const findResult = R.findIndex(
(row: { block_height: CassandraTypes.Long }) =>
row.block_height.equals(height)
)(blockQ.rows as { block_height: CassandraTypes.Long }[]);
if (findResult < 0) {
missingHeights.push(height);
}
}
}
return missingHeights;
}
return missingHeights;
};

/*
Expand Down