From 6f05e442ec5cd27b315b080dd1336b61243988fc Mon Sep 17 00:00:00 2001 From: neil-iohk Date: Wed, 16 Oct 2024 16:57:24 +0100 Subject: [PATCH] refactor: update sql schema autogen tool (#331) * chore(tooling): swap out dbviz custom database diagram tool for SchemaSpy, remove dependencies in catalyst-ci postgresql base * chore(tooling): swap out dbviz custom database diagram tool for SchemaSpy, remove dependencies in catalyst-ci postgresql base * chore(tooling): refactored how the dependencies are downloaded and cached to avoid apt install executing every run (reduced network calls) * chore(tooling): refactored how the dependencies are downloaded and cached to avoid apt install executing every run (reduced network calls) * chore(tooling): Fix markdown warnings breaking the CI build. No warnings required on md file as it is just a container for SchemaSpy * chore(tooling): fix breaking CI issues * chore(tooling): fix breaking CI issues * refactor(tooling): modified approach to align with repo goals based on PR feedback. Removed seperate layers for package installs * refactor(tooling): re-ordered package install and download steps for improved caching * refactor(tooling): removed dbviz utility from repository, dbviz is no longer required and has been replaced with SchemaSpy for database schema visualizations diagram generation --------- Co-authored-by: Steven Johnson --- .config/dictionaries/project.dic | 4 +- earthly/java/Earthfile | 15 + earthly/postgresql/Earthfile | 48 ++- earthly/postgresql/scripts/std_docs.py | 361 ++------------------- earthly/postgresql/templates/schema.md | 61 ++++ examples/postgresql/diagrams.json | 21 -- utilities/dbviz/.cargo/config.toml | 93 ------ utilities/dbviz/.config/nextest.toml | 49 --- utilities/dbviz/Cargo.toml | 53 --- utilities/dbviz/Earthfile | 49 --- utilities/dbviz/README.md | 11 - utilities/dbviz/clippy.toml | 2 - utilities/dbviz/cspell.json | 43 --- utilities/dbviz/deny.toml | 124 ------- utilities/dbviz/rust-toolchain.toml | 3 - utilities/dbviz/rustfmt.toml | 68 ---- utilities/dbviz/src/default_template.jinja | 222 ------------- utilities/dbviz/src/lib.rs | 2 - utilities/dbviz/src/main.rs | 39 --- utilities/dbviz/src/opts.rs | 74 ----- utilities/dbviz/src/postgresql.rs | 312 ------------------ utilities/dbviz/src/schema.rs | 84 ----- 22 files changed, 148 insertions(+), 1590 deletions(-) create mode 100644 earthly/java/Earthfile create mode 100644 earthly/postgresql/templates/schema.md delete mode 100644 examples/postgresql/diagrams.json delete mode 100644 utilities/dbviz/.cargo/config.toml delete mode 100644 utilities/dbviz/.config/nextest.toml delete mode 100644 utilities/dbviz/Cargo.toml delete mode 100644 utilities/dbviz/Earthfile delete mode 100644 utilities/dbviz/README.md delete mode 100644 utilities/dbviz/clippy.toml delete mode 100644 utilities/dbviz/cspell.json delete mode 100644 utilities/dbviz/deny.toml delete mode 100644 utilities/dbviz/rust-toolchain.toml delete mode 100644 utilities/dbviz/rustfmt.toml delete mode 100644 utilities/dbviz/src/default_template.jinja delete mode 100644 utilities/dbviz/src/lib.rs delete mode 100644 utilities/dbviz/src/main.rs delete mode 100644 utilities/dbviz/src/opts.rs delete mode 100644 utilities/dbviz/src/postgresql.rs delete mode 100644 utilities/dbviz/src/schema.rs diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic index ba465b0bb..bccaac193 100644 --- a/.config/dictionaries/project.dic +++ b/.config/dictionaries/project.dic @@ -95,6 +95,7 @@ rustflags rustfmt rustup sanchonet +schemaspy sqlfluff stdcfgs subproject @@ -120,4 +121,5 @@ WORKDIR xerrors xvfb zstd -zstdcat \ No newline at end of file +zstdcat +JDBC \ No newline at end of file diff --git a/earthly/java/Earthfile b/earthly/java/Earthfile new file mode 100644 index 000000000..98c8570aa --- /dev/null +++ b/earthly/java/Earthfile @@ -0,0 +1,15 @@ +VERSION 0.8 + +# Base image for Java used in other targets to avoid improve caching +java-base: + FROM openjdk:21-jdk-slim + + SAVE ARTIFACT /usr/local/openjdk-21 /java + +COPY_DEPS: + FUNCTION + COPY +java-base/java /usr/local/openjdk-21 + + # Set environment variables for Java + ENV JAVA_HOME=/usr/local/openjdk-21 + ENV PATH=$JAVA_HOME/bin:$PATH \ No newline at end of file diff --git a/earthly/postgresql/Earthfile b/earthly/postgresql/Earthfile index 37f75b076..611e09d62 100644 --- a/earthly/postgresql/Earthfile +++ b/earthly/postgresql/Earthfile @@ -2,14 +2,17 @@ VERSION 0.8 IMPORT ../rust/tools AS rust-tools -IMPORT ../../utilities/dbviz AS dbviz IMPORT ../../utilities/scripts AS scripts +IMPORT ../java AS java -# cspell: words psycopg dbviz +# cspell: words psycopg postgres-base: FROM postgres:16.4-bookworm + ARG SCHEMASPY_VERSION=6.2.4 + ARG POSTGRESQL_JDBC_VERSION=42.7.4 + WORKDIR /root # Install necessary packages @@ -22,7 +25,7 @@ postgres-base: colordiff \ findutils \ fontconfig \ - fonts-liberation2 \ + fonts-liberation2 \ graphviz \ libssl-dev \ mold \ @@ -36,17 +39,20 @@ postgres-base: pipx \ && rm -rf /var/lib/apt/lists/* + # Use the cached java installation from the java Earthfile + DO java+COPY_DEPS + # Install SQLFluff - Check ENV PATH="/root/.local/bin:${PATH}" RUN pipx install sqlfluff==3.1.1 && sqlfluff version + # Install SchemaSpy and required Postgresql JDBC driver + RUN wget -O /bin/postgresql.jar https://jdbc.postgresql.org/download/postgresql-${POSTGRESQL_JDBC_VERSION}.jar + RUN wget -O /bin/schemaspy.jar https://github.com/schemaspy/schemaspy/releases/download/v${SCHEMASPY_VERSION}/schemaspy-${SCHEMASPY_VERSION}.jar + # Get refinery COPY rust-tools+tool-refinery/refinery /bin - # Get dbviz - COPY dbviz+build/dbviz /bin - RUN dbviz --help - # Copy our set SQL files COPY --dir sql /sql @@ -56,7 +62,11 @@ postgres-base: DO scripts+ADD_BASH_SCRIPTS DO scripts+ADD_PYTHON_SCRIPTS + # Copy templates to the working directory + COPY --dir templates /templates + SAVE ARTIFACT /scripts /scripts + SAVE ARTIFACT /templates /templates # Common build setup steps. # Arguments: @@ -82,8 +92,13 @@ BUILDER: # DOCS - FUNCTION to build the docs, needs to be run INSIDE the BUILDER like so: # -# 1. Create a ./docs/diagrams.json which has the options needed to run to generate the docs to /docs -# 2. Define the following targets in your earthfile +# This function uses SchemaSpy to generate database documentation. +# SchemaSpy creates detailed, Discoverable ER diagrams and schema documentation. +# +# To use this function: +# 1. Ensure your migrations are in the ./migrations directory +# 2. Have a refinery.toml file to configure the migrations +# 3. Define the following targets in your earthfile: # # builder: # DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:+BUILDER --sqlfluff_cfg=./../../+repo-config/repo/.sqlfluff @@ -93,25 +108,27 @@ BUILDER: # # DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:+BUILD --image_name= # DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:+DOCS +# +# The generated documentation will be saved in the ./docs artifact. DOCS: FUNCTION - ARG diagrams=./diagrams.json ARG migrations=./migrations ARG refinery_toml=./refinery.toml + FROM +postgres-base + USER postgres:postgres WORKDIR /docs - COPY $diagrams ./diagrams.json COPY --dir $migrations . COPY --dir $refinery_toml . - RUN /scripts/std_docs.py ./diagrams.json - - SAVE ARTIFACT docs /docs - + RUN /scripts/std_docs.py + # Pull templates artifact from postgres-base + COPY +postgres-base/templates/schema.md ./docs/schema.md + SAVE ARTIFACT docs ./docs # Linter checks for sql files CHECK: @@ -171,3 +188,4 @@ BUILD: # Push the container... SAVE IMAGE ${image_name}:latest + diff --git a/earthly/postgresql/scripts/std_docs.py b/earthly/postgresql/scripts/std_docs.py index 09789a49c..e64c7e3ad 100755 --- a/earthly/postgresql/scripts/std_docs.py +++ b/earthly/postgresql/scripts/std_docs.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# cspell: words dbmigrations dbviz dbhost dbuser dbuserpw Tsvg +# cspell: words dbmigrations dbhost dbuser dbuserpw Tsvg pgsql11 from typing import Optional import python.exec_manager as exec_manager @@ -10,87 +10,14 @@ from rich import print import os import re -import json -from dataclasses import dataclass from textwrap import indent - -@dataclass -class DiagramCfg: - title: str - version: int - migration_name: str - tables: Optional[list[str]] - included_tables: Optional[list[str]] - excluded_tables: Optional[list[str]] - comments: Optional[bool] - column_description_wrap: Optional[int] - table_description_wrap: Optional[int] - sql_data: str - - def include( - self, - extra_includes: Optional[list[str]] = None, - extra_excludes: Optional[list[str]] = None, - ) -> Optional[list[str]]: - # We exclude from the global include tables, any tables the migration - # itself requests to be excluded. - - include_tables = self.included_tables if self.included_tables else [] - tables = self.tables if self.tables else [] - extra_includes = extra_includes if extra_includes else [] - excluded_tables = self.excluded_tables if self.excluded_tables else [] - extra_excludes = extra_excludes if extra_excludes else [] - - for table in tables + extra_includes: - if ( - table not in excluded_tables - and table not in extra_excludes - and table not in include_tables - ): - include_tables.append(table) - - if len(include_tables) == 0: - include_tables = None - - return include_tables - - def exclude( - self, extra_excludes: Optional[list[str]] = None - ) -> Optional[list[str]]: - # We exclude from the global exclude tables, any tables the migration - # specifically includes. - exclude_tables = self.excluded_tables if self.excluded_tables else [] - extra_excludes = extra_excludes if extra_excludes else [] - for table in extra_excludes: - if table not in exclude_tables: - exclude_tables.append(table) - - if len(exclude_tables) == 0: - exclude_tables = None - - return exclude_tables - - def process_sql_files(directory): file_pattern = r"V(\d+)__(\w+)\.sql" - table_pattern = r"CREATE TABLE(?: IF NOT EXISTS)? (\w+)" - - diagram_option_pattern = r"^--\s*(Title|Include|Exclude|Comment|Column Description Wrap|Table Description Wrap)\s+:\s*(.*)$" - migrations = {} largest_version = 0 for filename in os.listdir(directory): - clean_sql = "" - title = None - table_names = [] - included_tables = None - excluded_tables = None - comments = None - column_description_wrap = None - table_description_wrap = None - match = re.match(file_pattern, filename) if match: version = int(match.group(1)) @@ -101,61 +28,14 @@ def process_sql_files(directory): with open(os.path.join(directory, filename), "r") as file: sql_data = file.read() - for line in sql_data.splitlines(): - match = re.match(diagram_option_pattern, line) - if match: - if match.group(1).lower() == "title" and title is None: - title = match.group(2) - elif ( - match.group(1).lower() == "include" - and len(match.group(2)) > 0 - ): - if included_tables is None: - included_tables = [] - included_tables.append(match.group(2).split()) - elif ( - match.group(1).lower() == "exclude" - and len(match.group(2)) > 0 - ): - if excluded_tables is None: - excluded_tables = [] - excluded_tables.append(match.group(2).split()) - elif match.group(1).lower() == "comment": - if match.group(2).strip().lower() == "true": - comments = True - elif match.group(1).lower() == "column description wrap": - try: - column_description_wrap = int(match.group(2)) - except: - pass - elif match.group(1).lower() == "table description wrap": - try: - table_description_wrap = int(match.group(2)) - except: - pass - else: - # We strip diagram options from the SQL. - clean_sql += line + "\n" - - match = re.match(table_pattern, line) - if match: - table_names.append(match.group(1)) - - migrations[version] = DiagramCfg( - title, - version, - migration_name, - table_names, - included_tables, - excluded_tables, - comments, - column_description_wrap, - table_description_wrap, - clean_sql, - ) - return migrations, largest_version + migrations[version] = { + "version": version, + "migration_name": migration_name, + "sql_data": sql_data + } + return migrations, largest_version class Migrations: def __init__(self, args: argparse.Namespace): @@ -169,184 +49,8 @@ def __init__(self, args: argparse.Namespace): None """ self.args = args - - with open(args.diagram_config) as f: - self.config = json.load(f) - self.migrations, self.migration_version = process_sql_files(args.dbmigrations) - def schema_name(self) -> str: - return self.config.get("name", "Database Schema") - - def all_schema_comments(self) -> bool: - return self.config.get("all_schema", {}).get("comments", False) - - def full_schema_comments(self) -> bool: - return self.config.get("full_schema", {}).get( - "comments", self.all_schema_comments() - ) - - def all_schema_included_tables(self) -> list[str]: - return self.config.get("all_schema", {}).get("included_tables", []) - - def all_schema_excluded_tables(self) -> list[str]: - return self.config.get("all_schema", {}).get("excluded_tables", []) - - def full_schema_excluded_tables(self) -> list[str]: - return self.config.get("full_schema", {}).get( - "excluded_tables", self.all_schema_excluded_tables() - ) - - def all_schema_column_description_wrap(self) -> int: - return self.config.get("all_schema", {}).get("column_description_wrap", 50) - - def full_schema_column_description_wrap(self) -> int: - return self.config.get("full_schema", {}).get( - "column_description_wrap", self.all_schema_column_description_wrap() - ) - - def all_schema_table_description_wrap(self) -> int: - return self.config.get("all_schema", {}).get("table_description_wrap", 50) - - def full_schema_table_description_wrap(self) -> int: - return self.config.get("full_schema", {}).get( - "table_description_wrap", self.all_schema_table_description_wrap() - ) - - def dbviz( - self, - filename: str, - name: str, - title: str, - included_tables: Optional[list[str]] = None, - excluded_tables: Optional[list[str]] = None, - comments: Optional[bool] = None, - column_description_wrap: Optional[int] = None, - table_description_wrap: Optional[int] = None, - ) -> exec_manager.Result: - if len(title) > 0: - title = f' --title "{title}"' - - includes = "" - if included_tables: - for table in included_tables: - includes += f" -i {table}" - - excludes = "" - if excluded_tables: - for table in excluded_tables: - excludes += f" -e {table}" - - if comments: - comments = " --comments" - else: - comments = "" - - if column_description_wrap and column_description_wrap > 0: - column_description_wrap = ( - f" --column-description-wrap {column_description_wrap}" - ) - else: - column_description_wrap = "" - - if table_description_wrap and table_description_wrap > 0: - table_description_wrap = ( - f" --table-description-wrap {table_description_wrap}" - ) - else: - table_description_wrap = "" - - res = exec_manager.cli_run( - f"dbviz -d {self.args.dbname}" - + f" -h {self.args.dbhost}" - + f" -u {self.args.dbuser}" - + f" -p {self.args.dbuserpw}" - + f"{title}" - + f"{includes}" - + f"{excludes}" - + f"{comments}" - + f"{column_description_wrap}" - + f"{table_description_wrap}" - + f" | dot -Tsvg -o {filename}", - # + f" > {filename}.dot", - name=f"Generate Schema Diagram: {name}", - verbose=True, - ) - - # if res.ok: - # exec_manager.cli_run( - # f"dot -Tsvg {filename}.dot -o {filename}", - # name=f"Render Schema Diagram to SVG: {name}", - # verbose=True, - # ) - - return res - - def full_schema_diagram(self) -> exec_manager.Result: - # Create a full Schema Diagram. - return self.dbviz( - "docs/full-schema.svg", - "Full Schema", - self.schema_name(), - excluded_tables=self.full_schema_excluded_tables(), - comments=self.full_schema_comments(), - column_description_wrap=self.full_schema_column_description_wrap(), - table_description_wrap=self.full_schema_table_description_wrap(), - ) - - def migration_schema_diagram(self, ver: int) -> exec_manager.Result: - # Create a schema diagram for an individual migration. - if ver in self.migrations: - migration = self.migrations[ver] - - include_tables = migration.include( - self.all_schema_included_tables(), self.all_schema_excluded_tables() - ) - if include_tables is None: - return exec_manager.Result( - 0, - "", - "", - 0.0, - f"Migration {ver} has no tables to diagram.", - ) - - exclude_tables = migration.exclude(self.all_schema_excluded_tables()) - - title = f"{migration.migration_name}" - if migration.title and len(migration.title) > 0: - title = migration.title - - comments = None - if migration.comments is not None: - comments = migration.comments - else: - comments = self.all_schema_comments() - - return self.dbviz( - f"docs/migration-{ver}.svg", - f"V{ver}__{migration.migration_name}", - title, - included_tables=include_tables, - excluded_tables=exclude_tables, - comments=comments, - column_description_wrap=migration.column_description_wrap, - table_description_wrap=migration.table_description_wrap, - ) - - def create_diagrams(self, results: exec_manager.Results) -> exec_manager.Results: - # Create a full Schema Diagram first. - res = self.full_schema_diagram() - results.add(res) - - for ver in sorted(self.migrations.keys()): - res = self.migration_schema_diagram(ver) - results.add(res) - - # exec_manager.cli_run("ls -al docs", verbose=True) - - return results - def create_markdown_file(self, file_path): with open(file_path, "w") as markdown_file: # Write the title with the maximum migration version @@ -354,26 +58,13 @@ def create_markdown_file(self, file_path): "# Migrations (Version {}) \n\n".format(self.migration_version) ) - # Link the full schema diagram. - markdown_file.write('??? example "Full Schema Diagram"\n\n') - markdown_file.write( - ' ![Full Schema](./full-schema.svg "Full Schema")\n\n' - ) - # Write the contents of each file in order for version in sorted(self.migrations.keys()): migration = self.migrations[version] - sql_data = migration.sql_data.strip() + sql_data = migration["sql_data"].strip() # Write the title of the file - markdown_file.write(f"## {migration.migration_name}\n\n") - - if os.path.exists(f"docs/migration-{version}.svg"): - markdown_file.write('??? example "Schema Diagram"\n\n') - markdown_file.write( - f" ![Migration {migration.migration_name}]" - + f'(./migration-{version}.svg "{migration.migration_name}")\n\n' - ) + markdown_file.write(f"## {migration['migration_name']}\n\n") markdown_file.write('??? abstract "Schema Definition"\n\n') markdown_file.write( @@ -382,7 +73,6 @@ def create_markdown_file(self, file_path): print("Markdown file created successfully at: {}".format(file_path)) - def main(): # Force color output in CI rich.reconfigure(color_system="256") @@ -390,7 +80,6 @@ def main(): parser = argparse.ArgumentParser( description="Standard Postgresql Documentation Processing." ) - parser.add_argument("diagram_config", help="Diagram Configuration JSON") parser.add_argument("--verbose", action="store_true", help="Enable verbose output") db_ops.add_args(parser) @@ -418,21 +107,43 @@ def main(): results.add(res) if res.ok(): - exec_manager.cli_run("mkdir docs") # Where we build the docs. + # Create the docs directory + exec_manager.cli_run("mkdir -p docs") # Where we build the docs. # Get all info about the migrations. migrations = Migrations(args) - results = migrations.create_diagrams(results) if results.ok(): + schemaspy_cmd = ( + f"java -jar /bin/schemaspy.jar -t pgsql11 " + f"-dp /bin/postgresql.jar " + f"-db {args.dbname} " + f"-host {args.dbhost} " + f"-u {args.dbuser} " + f"-p {args.dbuserpw} " + f"-o docs/database_schema/ " + ) + res = exec_manager.cli_run( + schemaspy_cmd, + name="Generate SchemaSpy Documentation", + verbose=True + ) + results.add(res) + + # If SchemaSpy command completes without error, create .pages file to hide the schema folder + if res.ok(): + exec_manager.cli_run( + 'echo "hide: true" > docs/database_schema/.pages', + name="Create .pages file", + verbose=True + ) + migrations.create_markdown_file("docs/migrations.md") - # exec_manager.cli_run("cat /tmp/migrations.md", verbose=True) results.print() if not results.ok(): exit(1) - if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/earthly/postgresql/templates/schema.md b/earthly/postgresql/templates/schema.md new file mode 100644 index 000000000..db40d5efd --- /dev/null +++ b/earthly/postgresql/templates/schema.md @@ -0,0 +1,61 @@ +--- +icon: material/database +hide: + - navigation + - toc +--- + + + + + + + + diff --git a/examples/postgresql/diagrams.json b/examples/postgresql/diagrams.json deleted file mode 100644 index 62e87058b..000000000 --- a/examples/postgresql/diagrams.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "Example Postgresql Database", - "all_schema": { - "comments": true, - "included_tables": [], - "excluded_tables": [ - "refinery_schema_history" - ], - "column_description_wrap": 50, - "table_description_wrap": 50 - }, - "full_schema": { - "excluded_tables": [ - "refinery_schema_history" - ], - "title": "Full Schema", - "comments": true, - "column_description_wrap": 50, - "table_description_wrap": 50 - } -} \ No newline at end of file diff --git a/utilities/dbviz/.cargo/config.toml b/utilities/dbviz/.cargo/config.toml deleted file mode 100644 index 02c231407..000000000 --- a/utilities/dbviz/.cargo/config.toml +++ /dev/null @@ -1,93 +0,0 @@ -# Use MOLD linker where possible, but ONLY in CI applicable targets. - -# Configure how Docker container targets build. - -# If you want to customize these targets for a local build, then customize them in your: -# $CARGO_HOME/config.toml -# NOT in the project itself. -# These targets are ONLY the targets used by CI and inside docker builds. - -# DO NOT remove `"-C", "target-feature=+crt-static"` from the rustflags for these targets. - -# Should be the default to have fully static rust programs in CI -[target.x86_64-unknown-linux-musl] -linker = "clang" -rustflags = [ - "-C", "link-arg=-fuse-ld=/usr/bin/mold", - "-C", "target-feature=-crt-static" -] - -# Should be the default to have fully static rust programs in CI -[target.aarch64-unknown-linux-musl] -linker = "clang" -rustflags = [ - "-C", "link-arg=-fuse-ld=/usr/bin/mold", - "-C", "target-feature=-crt-static" -] - -[build] -rustflags = [] -rustdocflags = [ - "--enable-index-page", - "-Z", - "unstable-options", -] - -[profile.dev] -opt-level = 1 -debug = true -debug-assertions = true -overflow-checks = true -lto = false -panic = "unwind" -incremental = true -codegen-units = 256 - -[profile.release] -opt-level = 3 -debug = false -debug-assertions = false -overflow-checks = false -lto = "thin" -panic = "unwind" -incremental = false -codegen-units = 16 - -[profile.test] -opt-level = 3 -debug = true -lto = false -debug-assertions = true -incremental = true -codegen-units = 256 - -[profile.bench] -opt-level = 3 -debug = false -debug-assertions = false -overflow-checks = false -lto = "thin" -incremental = false -codegen-units = 16 - -[alias] -lint = "clippy --all-targets" -lintfix = "clippy --all-targets --fix --allow-dirty" -lint-vscode = "clippy --message-format=json-diagnostic-rendered-ansi --all-targets" - -docs = "doc --release --no-deps --document-private-items --bins --lib --examples" -# nightly docs build broken... when they are'nt we can enable these docs... --unit-graph --timings=html,json -Z unstable-options" -testunit = "nextest run --release --bins --lib --tests --benches --no-fail-fast -P ci" -testcov = "llvm-cov nextest --release --bins --lib --tests --benches --no-fail-fast -P ci" -testdocs = "test --doc --release" - -# Rust formatting, MUST be run with +nightly -fmtchk = "fmt -- --check -v --color=always" -fmtfix = "fmt -- -v" - -[term] -quiet = false # whether cargo output is quiet -verbose = false # whether cargo provides verbose output -color = "auto" # whether cargo colorizes output use `CARGO_TERM_COLOR="off"` to disable. -progress.when = "never" # whether cargo shows progress bar -progress.width = 80 # width of progress bar diff --git a/utilities/dbviz/.config/nextest.toml b/utilities/dbviz/.config/nextest.toml deleted file mode 100644 index be3673830..000000000 --- a/utilities/dbviz/.config/nextest.toml +++ /dev/null @@ -1,49 +0,0 @@ -# cspell: words scrollability testcase -[store] -# The directory under the workspace root at which nextest-related files are -# written. Profile-specific storage is currently written to dir/. -# dir = "target/nextest" - -[profile.default] -# Print out output for failing tests as soon as they fail, and also at the end -# of the run (for easy scrollability). -failure-output = "immediate-final" - -# Do not cancel the test run on the first failure. -fail-fast = true - -status-level = "all" -final-status-level = "all" - -[profile.ci] -# Print out output for failing tests as soon as they fail, and also at the end -# of the run (for easy scrollability). -failure-output = "immediate-final" -# Do not cancel the test run on the first failure. -fail-fast = false - -status-level = "all" -final-status-level = "all" - - -[profile.ci.junit] -# Output a JUnit report into the given file inside 'store.dir/'. -# If unspecified, JUnit is not written out. - -path = "junit.xml" - -# The name of the top-level "report" element in JUnit report. If aggregating -# reports across different test runs, it may be useful to provide separate names -# for each report. -report-name = "nextest" - -# Whether standard output and standard error for passing tests should be stored in the JUnit report. -# Output is stored in the and elements of the element. -store-success-output = true - -# Whether standard output and standard error for failing tests should be stored in the JUnit report. -# Output is stored in the and elements of the element. -# -# Note that if a description can be extracted from the output, it is always stored in the -# element. -store-failure-output = true diff --git a/utilities/dbviz/Cargo.toml b/utilities/dbviz/Cargo.toml deleted file mode 100644 index c1c58329b..000000000 --- a/utilities/dbviz/Cargo.toml +++ /dev/null @@ -1,53 +0,0 @@ -[package] -name = "dbviz" -version = "1.4.0" -authors = ["Steven Johnson"] -edition = "2021" -license = "Apache-2.0/MIT" - -[lints.rust] -warnings = "deny" -missing_docs = "deny" -let_underscore_drop = "deny" -non_ascii_idents = "deny" -single_use_lifetimes = "deny" -trivial_casts = "deny" -trivial_numeric_casts = "deny" - -[lints.rustdoc] -broken_intra_doc_links = "deny" -invalid_codeblock_attributes = "deny" -invalid_html_tags = "deny" -invalid_rust_codeblocks = "deny" -bare_urls = "deny" -unescaped_backticks = "deny" - -[lints.clippy] -pedantic = { level = "deny", priority = -1 } -unwrap_used = "deny" -expect_used = "deny" -todo = "deny" -unimplemented = "deny" -exit = "deny" -get_unwrap = "deny" -index_refutable_slice = "deny" -indexing_slicing = "deny" -match_on_vec_items = "deny" -match_wild_err_arm = "deny" -missing_panics_doc = "deny" -panic = "deny" -string_slice = "deny" -unchecked_duration_subtraction = "deny" -unreachable = "deny" -missing_docs_in_private_items = "deny" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -anyhow = "1.0" -itertools = "0.12.0" -postgres = "0.19.4" -minijinja = "1.0.10" -textwrap = "0.16.0" -serde = { version = "1.0", features = ["derive"] } -clap = { version = "4.4.8", features = ["derive"] } diff --git a/utilities/dbviz/Earthfile b/utilities/dbviz/Earthfile deleted file mode 100644 index efdd1d336..000000000 --- a/utilities/dbviz/Earthfile +++ /dev/null @@ -1,49 +0,0 @@ -VERSION 0.8 - -IMPORT ./../../earthly/rust AS rust-ci - -# cspell: words stdcfgs toolset toolsets - -# Internal: Set up our target toolchains, and copy our files. -builder: - DO rust-ci+SETUP - - COPY --keep-ts --dir \ - .cargo .config Cargo.toml clippy.toml deny.toml rustfmt.toml \ - src \ - . - -## ----------------------------------------------------------------------------- -## -## Standard CI targets. -## -## These targets are discovered and executed automatically by CI. - -# check - Run check using the most efficient host tooling -# CI Automated Entry point. -check: - FROM +builder - - DO rust-ci+EXECUTE --cmd="/scripts/std_checks.py" - -# Test which runs check with all supported host tooling. Needs qemu or rosetta to run. -# Only used to validate tooling is working across host toolsets. -all-hosts-check: - BUILD --platform=linux/amd64 --platform=linux/arm64 +check - -# build - Run build using the most efficient host tooling -# CI Automated Entry point. -build: - FROM +builder - - DO rust-ci+EXECUTE \ - --cmd="/scripts/std_build.py --bins=dbviz/dbviz" \ - --output="release/[^\./]+" \ - --docs="true" - - SAVE ARTIFACT target/release/dbviz dbviz - -# Test which runs check with all supported host tooling. Needs qemu or rosetta to run. -# Only used to validate tooling is working across host toolsets. -all-hosts-build: - BUILD --platform=linux/amd64 --platform=linux/arm64 +build diff --git a/utilities/dbviz/README.md b/utilities/dbviz/README.md deleted file mode 100644 index 07f40b1ad..000000000 --- a/utilities/dbviz/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# dbviz - -Simple tool to create database diagrams from postgres schemas. -The diagrams themselves are just text files, and are controlled by `.jinja` templates. -The tool builds in a default template which produces `.dot` files. - -## Usage - -```sh -dbviz -d database_name | dot -Tpng > schema.png -``` diff --git a/utilities/dbviz/clippy.toml b/utilities/dbviz/clippy.toml deleted file mode 100644 index 0358cdb50..000000000 --- a/utilities/dbviz/clippy.toml +++ /dev/null @@ -1,2 +0,0 @@ -allow-unwrap-in-tests = true -allow-expect-in-tests = true diff --git a/utilities/dbviz/cspell.json b/utilities/dbviz/cspell.json deleted file mode 100644 index a45613263..000000000 --- a/utilities/dbviz/cspell.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json", - "version": "0.2", - "import": "../../cspell.json", - "words": [ - "dbviz", - "Tpng", - "dbname", - "regclass", - "nspname", - "schemaname", - "relname", - "attname", - "attrf", - "conrelid", - "attnum", - "conkey", - "relnamespace", - "confrelid", - "attrelid", - "confkey", - "indrelid", - "indisprimary", - "indisunique", - "indexdef", - "indexrelid", - "indkey", - "relam", - "minijinja", - "pkey", - "endmacro", - "varchar", - "startingwith", - "nextval", - "endmacro", - "labelloc", - "rankdir", - "cellborder", - "endfor", - "bytea", - "Autoincrement" - ] -} \ No newline at end of file diff --git a/utilities/dbviz/deny.toml b/utilities/dbviz/deny.toml deleted file mode 100644 index 77f0259f1..000000000 --- a/utilities/dbviz/deny.toml +++ /dev/null @@ -1,124 +0,0 @@ -# cspell: words msvc, wasip, RUSTSEC, rustls, libssh, reqwest, tinyvec, Leay, webpki - -[graph] -# cargo-deny is really only ever intended to run on the "normal" tier-1 targets -targets = [ - "x86_64-unknown-linux-gnu", - "aarch64-unknown-linux-gnu", - "x86_64-unknown-linux-musl", - "aarch64-apple-darwin", - "x86_64-apple-darwin", - "x86_64-pc-windows-msvc", - "wasm32-unknown-unknown", - "wasm32-wasip1", - "wasm32-wasip2", -] - -[advisories] -version = 2 -ignore = [ - { id = "RUSTSEC-2020-0168", reason = "`mach` is used by wasmtime and we have no control over that." }, - { id = "RUSTSEC-2021-0145", reason = "we don't target windows, and don't use a custom global allocator." }, - { id = "RUSTSEC-2024-0370", reason = "`proc-macro-error` is used by crates we rely on, we can't control what they use."}, -] - -[bans] -multiple-versions = "warn" -wildcards = 'deny' -deny = [ - # Scylla DB Drivers currently require OpenSSL. Its unavoidable. - # However, there is movement to enable support for Rustls. - # So, for now, allow open-ssl but it needs to be disabled as soon as Scylla DB enables Rustls. - #{ crate = "openssl", use-instead = "rustls" }, - #{ crate = "openssl-sys", use-instead = "rustls" }, - "libssh2-sys", - # { crate = "git2", use-instead = "gix" }, - # { crate = "cmake", use-instead = "cc" }, - # { crate = "windows", reason = "bloated and unnecessary", use-instead = "ideally inline bindings, practically, windows-sys" }, -] -skip = [ - # { crate = "bitflags@1.3.2", reason = "https://github.com/seanmonstar/reqwest/pull/2130 should be in the next version" }, - # { crate = "winnow@0.5.40", reason = "gix 0.59 was yanked, see https://github.com/Byron/gitoxide/issues/1309" }, - # { crate = "heck@0.4.1", reason = "strum_macros uses this old version" }, - # { crate = "base64@0.21.7", reason = "gix-transport pulls in this old version, as well as a newer version via reqwest" }, - # { crate = "byte-array-literalsase64@0.21.7", reason = "gix-transport pulls in this old version, as well as a newer version via reqwest" }, -] -skip-tree = [ - { crate = "windows-sys@0.48.0", reason = "a foundational crate for many that bumps far too frequently to ever have a shared version" }, -] - -[sources] -unknown-registry = "deny" -unknown-git = "deny" - -# List of URLs for allowed Git repositories -allow-git = [ - "https://github.com/input-output-hk/catalyst-libs.git", - "https://github.com/input-output-hk/catalyst-pallas.git", - "https://github.com/input-output-hk/catalyst-mithril.git", - "https://github.com/bytecodealliance/wasmtime", - "https://github.com/aldanor/hdf5-rust", -] - -[licenses] -version = 2 -# Don't warn if a listed license isn't found -unused-allowed-license="allow" -# We want really high confidence when inferring licenses from text -confidence-threshold = 0.93 -allow = [ - "MIT", - "Apache-2.0", - "Unicode-DFS-2016", - "BSD-3-Clause", - "BSD-2-Clause", - "BlueOak-1.0.0", - "Apache-2.0 WITH LLVM-exception", - "CC0-1.0", - "ISC", - "Unicode-3.0", - "MPL-2.0", - "Zlib", - "MIT-0", -] -exceptions = [ - #{ allow = ["Zlib"], crate = "tinyvec" }, - #{ allow = ["Unicode-DFS-2016"], crate = "unicode-ident" }, - #{ allow = ["OpenSSL"], crate = "ring" }, -] - -[[licenses.clarify]] -crate = "byte-array-literals" -expression = "Apache-2.0 WITH LLVM-exception" -license-files = [{ path = "../../../LICENSE", hash = 0x001c7e6c }] - -[[licenses.clarify]] -crate = "hdf5-src" -expression = "MIT" -license-files = [{ path = "../LICENSE-MIT", hash = 0x001c7e6c }] - -[[licenses.clarify]] -crate = "ring" -expression = "MIT" -license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }] - -# SPDX considers OpenSSL to encompass both the OpenSSL and SSLeay licenses -# https://spdx.org/licenses/OpenSSL.html -# ISC - Both BoringSSL and ring use this for their new files -# MIT - "Files in third_party/ have their own licenses, as described therein. The MIT -# license, for third_party/fiat, which, unlike other third_party directories, is -# compiled into non-test libraries, is included below." -# OpenSSL - Obviously -#expression = "ISC AND MIT AND OpenSSL" -#license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }] - -#[[licenses.clarify]] -#crate = "webpki" -#expression = "ISC" -#license-files = [{ path = "LICENSE", hash = 0x001c7e6c }] - -# Actually "ISC-style" -#[[licenses.clarify]] -#crate = "rustls-webpki" -#expression = "ISC" -#license-files = [{ path = "LICENSE", hash = 0x001c7e6c }] diff --git a/utilities/dbviz/rust-toolchain.toml b/utilities/dbviz/rust-toolchain.toml deleted file mode 100644 index f01d02df3..000000000 --- a/utilities/dbviz/rust-toolchain.toml +++ /dev/null @@ -1,3 +0,0 @@ -[toolchain] -channel = "1.81" -profile = "default" \ No newline at end of file diff --git a/utilities/dbviz/rustfmt.toml b/utilities/dbviz/rustfmt.toml deleted file mode 100644 index b0f20832c..000000000 --- a/utilities/dbviz/rustfmt.toml +++ /dev/null @@ -1,68 +0,0 @@ -# Enable unstable features: -# * imports_indent -# * imports_layout -# * imports_granularity -# * group_imports -# * reorder_impl_items -# * trailing_comma -# * where_single_line -# * wrap_comments -# * comment_width -# * blank_lines_upper_bound -# * condense_wildcard_suffixes -# * force_multiline_blocks -# * format_code_in_doc_comments -# * format_generated_files -# * hex_literal_case -# * inline_attribute_width -# * normalize_comments -# * normalize_doc_attributes -# * overflow_delimited_expr -unstable_features = true - -# Compatibility: -edition = "2021" - -# Tabs & spaces - Defaults, listed for clarity -tab_spaces = 4 -hard_tabs = false - -# Commas. -trailing_comma = "Vertical" -match_block_trailing_comma = true - -# General width constraints. -max_width = 100 - -# Comments: -normalize_comments = true -normalize_doc_attributes = true -wrap_comments = true -comment_width = 90 # small excess is okay but prefer 80 -format_code_in_doc_comments = true -format_generated_files = false - -# Imports. -imports_indent = "Block" -imports_layout = "Mixed" -group_imports = "StdExternalCrate" -reorder_imports = true -imports_granularity = "Crate" - -# Arguments: -use_small_heuristics = "Default" -fn_params_layout = "Compressed" -overflow_delimited_expr = true -where_single_line = true - -# Misc: -inline_attribute_width = 0 -blank_lines_upper_bound = 1 -reorder_impl_items = true -use_field_init_shorthand = true -force_multiline_blocks = true -condense_wildcard_suffixes = true -hex_literal_case = "Upper" - -# Ignored files: -ignore = [] diff --git a/utilities/dbviz/src/default_template.jinja b/utilities/dbviz/src/default_template.jinja deleted file mode 100644 index 64488162b..000000000 --- a/utilities/dbviz/src/default_template.jinja +++ /dev/null @@ -1,222 +0,0 @@ -{%- macro theme(x) -%} -{{ { - "title_loc" : "t", - "title_size" : 30, - "title_color" : "blue", - "graph_direction" : "LR", - "default_fontsize" : 16, - "pkey_bgcolor" : "seagreen1", - "field_desc" : "color='grey50' face='Monospace' point-size='14'", - "column_heading" : "color='black' face='Courier bold' point-size='18'", - "table_heading_bgcolor" : "#009879", - "table_heading" : "color='white' face='Courier bold italic' point-size='20'", - "table_desc_bgcolor" : "grey20", - "table_description" : "color='white' face='Monospace' point-size='14'" -}[x] }} -{%- endmacro -%} - -{%- macro data_type(data_type, default, max_chars, nullable) -%} - - {%- if nullable == "YES" -%} - - {%- endif -%} - - {%- if data_type == "character varying" -%} - varchar - {%- elif data_type == "timestamp without time zone" -%} - timestamp - {%- else -%} - {{- data_type -}} - {%- endif -%} - - {%- if max_chars is not none -%} - ({{max_chars}}) - {%- endif -%} - - {%- if default is startingwith("nextval") -%} - + - {%- endif -%} - - {%- if nullable == "YES" -%} - - {%- endif -%} - -{%- endmacro -%} - -{%- macro pkey_bgcolor(pkey) -%} - {%- if pkey -%} - bgcolor="{{ theme('pkey_bgcolor') }}" - {%- endif -%} -{%- endmacro -%} - -{%- macro column_name_x(name, pkey) -%} - {{name}}
-{%- endmacro -%} - - -{%- macro column_name(field) -%} - {{- column_name_x(field.column, field.primary_key) -}} -{%- endmacro -%} - -{%- macro final(field, final=true) -%} - {%- if final -%} - port="{{field.column}}_out" - {%- endif -%} -{%- endmacro -%} - -{%- macro column_type(field, last=false) -%} - {{ data_type(field.data_type, field.default, field.max_chars, field.nullable ) }} -{%- endmacro -%} - -{%- macro column_description(field) -%} - {{- field.description|trim|escape|replace("\n", "
") -}}
 -{%- endmacro -%} - -digraph erd { - - {% if opts.title is not none %} - label = "{{ opts.title }}" - labelloc = {{ theme("title_loc") }} - fontsize = {{ theme("title_size") }} - fontcolor = {{ theme("title_color") }} - {% endif %} - - graph [ - rankdir = "{{theme('graph_direction')}}" - ]; - - node [ - fontsize = "{{theme('default_fontsize')}}" - shape = "plaintext" - ]; - - edge [ - ]; - - {% for table in schema.tables %} - {% if opts.comments %} - - "{{table.name}}" [shape=plain label=< - - - - - - - - - - - {% for field in table.fields %} - - {{ column_name(field) }} - {{ column_type(field, false) }} - {{ column_description(field) }} - - {% endfor %} - - {% if table.description is not none %} - - - - {% endif %} - -
{{table.name}}
ColumnTypeDescription
{{- table.description|trim|escape|replace("\n", "
") -}}
- >]; - - {% else %} - - "{{table.name}}" [label=< - - - - - - - - - - {% for field in table.fields %} - - {{ column_name(field) }} - {{ column_type(field, true) }} - - {% endfor %} - -
{{table.name}}
ColumnType
- >]; - - {% endif %} - {% endfor %} - - {% for partial in schema.partial_tables %} - - "{{partial}}" [label=< - - - - - - - - - {% for field in schema.partial_tables[partial] %} - - {{ column_name_x(field,false) }} - - {% endfor %} - - - - -
{{partial}}
Column
ABRIDGED
- >]; - - {% endfor %} - - - "LEGEND" [label=< - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
LEGEND
TypeExample
Primary Key
{{ data_type("integer", "nextval", none, "NO") }}
Standard Field
{{ data_type("bytea", none, none, "NO") }}
Nullable Field
{{ data_type("text", none, none, "YES") }}
Sized Field
{{ data_type("varchar", none, 32, "NO") }}
Autoincrement Field
{{ data_type("integer", "nextval", none, "NO") }}
- >]; - - {% for relation in schema.relations %} - "{{relation.on_table}}":"{{relation.on_field}}_out" -> "{{relation.to_table}}":"{{relation.to_field}}" - {% endfor %} - - -} diff --git a/utilities/dbviz/src/lib.rs b/utilities/dbviz/src/lib.rs deleted file mode 100644 index 2a0f48add..000000000 --- a/utilities/dbviz/src/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ -//! Intentionally empty -//! This file exists, so that doc tests can be used inside binary crates. diff --git a/utilities/dbviz/src/main.rs b/utilities/dbviz/src/main.rs deleted file mode 100644 index a22d08ac4..000000000 --- a/utilities/dbviz/src/main.rs +++ /dev/null @@ -1,39 +0,0 @@ -//! `DBViz` - Database Diagram Generator -//! -//! `DBViz` is a tool for generating database diagrams. - -mod opts; -mod postgresql; -mod schema; - -use std::fs; - -use anyhow::Result; -use minijinja::{context, Environment}; - -fn main() -> Result<()> { - let opts = opts::load(); - - let loader = postgresql::Conn::new(&opts)?; - let schema = loader.load()?; - - let template_file = match &opts.template { - Some(fname) => fs::read_to_string(fname)?, - None => include_str!("default_template.jinja").to_string(), - }; - - let mut env = Environment::new(); - env.add_template("diagram", &template_file)?; - let tmpl = env.get_template("diagram")?; - - let ctx = context!( - opts => opts, - schema => schema - ); - - let rendered = tmpl.render(ctx)?; - - println!("{rendered}"); - - Ok(()) -} diff --git a/utilities/dbviz/src/opts.rs b/utilities/dbviz/src/opts.rs deleted file mode 100644 index 43d38d13d..000000000 --- a/utilities/dbviz/src/opts.rs +++ /dev/null @@ -1,74 +0,0 @@ -//! CLI Option parsing - -use std::path::PathBuf; - -use clap::{Args, Parser}; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Parser, Clone, Serialize, Deserialize)] -#[command(author, version, about, long_about = None)] -/// `DBViz` a tool for generating database diagrams. -pub(crate) struct Cli { - #[command(flatten)] - /// Postgres connection options - pub(crate) pg_opts: Pg, - - #[arg(short, long)] - /// Tables to include in the current diagram. - pub(crate) include_tables: Option>, - - #[arg(short, long)] - /// Tables to completely exclude in the current diagram. - pub(crate) exclude_tables: Option>, - - /// Title to give the Diagram - #[arg(short, long)] - pub(crate) title: Option, - - /// How wide is the Column Description before we wrap it? - #[arg(long)] - pub(crate) column_description_wrap: Option, - - /// How wide is the Table Description before we wrap it? - #[arg(long)] - pub(crate) table_description_wrap: Option, - - /// Do we include comments in the diagram? - #[arg(long)] - pub(crate) comments: bool, - - /// Input file - pub(crate) template: Option, - - /// Output file - pub(crate) output: Option, -} - -#[derive(Debug, Args, Clone, Serialize, Deserialize)] -/// Postgres connection options -pub(crate) struct Pg { - #[arg(short, long, default_value = "localhost")] - /// Hostname to connect to - pub(crate) hostname: String, - - #[arg(short, long, default_value = "postgres")] - /// Username to use when connecting - pub(crate) username: String, - - #[arg(short, long, default_value = "postgres")] - /// Password to use when connecting - pub(crate) password: String, - - #[arg(short, long, default_value = "postgres")] - /// Database name to connect to - pub(crate) database: String, - - #[arg(short, long, default_value = "public")] - /// Schema name to use - pub(crate) schema: String, -} - -/// Load CLI Options. -pub(crate) fn load() -> Cli { - Cli::parse() -} diff --git a/utilities/dbviz/src/postgresql.rs b/utilities/dbviz/src/postgresql.rs deleted file mode 100644 index 297eea796..000000000 --- a/utilities/dbviz/src/postgresql.rs +++ /dev/null @@ -1,312 +0,0 @@ -//! Loader for postgresql. - -// This is not production code, so while we should get rid of these lints, its not -// worth the time it would take, for no return value. -#![allow(clippy::unwrap_used)] -#![allow(clippy::indexing_slicing)] - -use std::{ - cell::RefCell, - collections::HashMap, - convert::{TryFrom, TryInto}, -}; - -use anyhow::Result; -use itertools::Itertools; -use postgres::{tls::NoTls, Client, Row}; - -use crate::{ - opts, - schema::{Index, Relation, Schema, Table, TableColumn}, -}; - -/// Struct that manages the loading and implements `Loader` trait. -pub struct Conn { - /// Postgres client - pg_client: RefCell, - /// Schema name - schema: String, - /// Options - opts: opts::Cli, -} - -/// Check if a column is a primary key -fn is_primary_key(table: &str, column: &str, indexes: &[Index]) -> bool { - indexes - .iter() - .any(|idx| idx.table == table && idx.fields.contains(&column.to_string()) && idx.primary) -} - -impl Conn { - /// Make a new postgres connection - pub(crate) fn new(opts: &opts::Cli) -> Result { - let pg_client = postgres::Config::new() - .user(&opts.pg_opts.username) - .password(&opts.pg_opts.password) - .dbname(&opts.pg_opts.database) - .host(&opts.pg_opts.hostname) - .connect(NoTls)?; - - let pg_client = RefCell::new(pg_client); - let schema = opts.pg_opts.schema.clone(); - Ok(Conn { - pg_client, - schema, - opts: opts.clone(), - }) - } - - /// Do we include this table name? - fn include_table(&self, name: &String) -> bool { - match &self.opts.include_tables { - Some(inc) => inc.contains(name), - None => true, - } - } - - /// Do we exclude this table name? - fn exclude_table(&self, name: &String) -> bool { - match &self.opts.exclude_tables { - Some(inc) => inc.contains(name), - None => false, - } - } - - /// Load the schema - pub(crate) fn load(&self) -> Result { - let mut client = self.pg_client.borrow_mut(); - let tables_rows = client.query(tables_query(), &[&self.schema])?; - let relations_rows = client.query(relations_query(), &[&self.schema])?; - let index_rows = client.query(index_query(), &[])?; - - let mut partial_tables: HashMap> = HashMap::new(); - - let indexes: Vec<_> = index_rows - .into_iter() - .filter(|row| { - let row_name: String = row.get(0); - self.include_table(&row_name) && !self.exclude_table(&row_name) - }) - .map(|row| { - let idx: Index = row.try_into().unwrap(); - idx - }) - .collect(); - - let tables: Vec<_> = tables_rows - .into_iter() - .group_by(|row| row.get(0)) - .into_iter() - .filter(|(name, _rows)| self.include_table(name) && !self.exclude_table(name)) - .map(|(name, rows)| { - let fields: Vec<_> = rows - .into_iter() - .map(|row| { - let mut field: TableColumn = row.try_into().unwrap(); - field.primary_key = is_primary_key(&name, &field.column, &indexes); - - let desc = match field.description { - Some(desc) => { - match self.opts.column_description_wrap { - Some(wrap) => Some(textwrap::fill(&desc, wrap)), - None => Some(desc), - } - }, - None => None, - }; - field.description = desc; - - field - }) - .collect(); - - let desc = match &fields[0].table_description { - Some(desc) => { - match self.opts.table_description_wrap { - Some(wrap) => Some(textwrap::fill(desc, wrap)), - None => Some(desc).cloned(), - } - }, - None => None, - }; - - Table { - name, - description: desc, - fields, - } - }) - .collect(); - - let relations: Vec<_> = relations_rows - .into_iter() - .map(|row| { - let relation: Relation = row.try_into().unwrap(); - relation - }) - .filter(|relation| { - if self.include_table(&relation.on_table) - && !self.exclude_table(&relation.on_table) - && !self.exclude_table(&relation.to_table) - { - if !self.include_table(&relation.to_table) { - match partial_tables.get_mut(&relation.to_table) { - Some(value) => { - if !value.contains(&relation.to_field) { - value.push(relation.to_field.clone()); - } - }, - None => { - partial_tables.insert(relation.to_table.clone(), vec![relation - .to_field - .clone()]); - }, - } - } - true - } else { - false - } - }) - .collect(); - - Ok(Schema { - tables, - relations, - partial_tables, - }) - } -} - -impl TryFrom for Index { - type Error = String; - - fn try_from(row: Row) -> std::result::Result { - let all_fields: String = row.get(4); - let braces: &[_] = &['{', '}']; - - let fields: Vec<_> = all_fields - .trim_matches(braces) - .split(',') - .map(std::string::ToString::to_string) - .collect(); - - Ok(Self { - table: row.get(0), - // name: row.get(1), - primary: row.get(2), - // unique: row.get(3), - fields, - }) - } -} - -impl TryFrom for Relation { - type Error = String; - - fn try_from(row: Row) -> std::result::Result { - let fields: HashMap = row - .columns() - .iter() - .enumerate() - .map(|(i, c)| (c.name().to_string(), row.get(i))) - .collect(); - - Ok(Self { - on_table: fetch_field(&fields, "on_table")?, - on_field: fetch_field(&fields, "on_field")?, - to_table: fetch_field(&fields, "to_table")?, - to_field: fetch_field(&fields, "to_field")?, - }) - } -} - -impl TryFrom for TableColumn { - type Error = String; - - fn try_from(row: Row) -> std::result::Result { - Ok(Self { - column: row.get(1), - data_type: row.get(2), - index: row.get(3), - default: row.get(4), - nullable: row.get(5), - max_chars: row.get(6), - description: row.get(7), - table_description: row.get(8), - primary_key: false, - }) - } -} - -/// Fetch a field from a hashmap -fn fetch_field(map: &HashMap, key: &str) -> std::result::Result { - map.get(key) - .cloned() - .ok_or(format!("could not find field {key}")) -} - -/// Query all tables and columns -fn tables_query() -> &'static str { - " - select table_name, column_name, data_type, ordinal_position, column_default, is_nullable, character_maximum_length, col_description(table_name::regclass, ordinal_position), obj_description(table_name::regclass) - from information_schema.columns - where table_schema = $1 - order by table_name, ordinal_position - " -} - -/// Query all relationships -fn relations_query() -> &'static str { - " - select * - from ( - select ns.nspname AS schemaname, - cl.relname AS on_table, - attr.attname AS on_field, - clf.relname AS to_table, - attrf.attname AS to_field - from pg_constraint con - join pg_class cl - on con.conrelid = cl.oid - join pg_namespace ns - on cl.relnamespace = ns.oid - join pg_class clf - on con.confrelid = clf.oid - join pg_attribute attr - on attr.attnum = ANY(con.conkey) and - attr.attrelid = con.conrelid - join pg_attribute attrf - on attrf.attnum = ANY(con.confkey) and - attrf.attrelid = con.confrelid - ) as fk - where fk.schemaname = $1 - " -} - -/// Query all indexes -fn index_query() -> &'static str { - " -SELECT - CAST(idx.indrelid::regclass as varchar) as table_name, - i.relname as index_name, - idx.indisprimary as primary_key, - idx.indisunique as unique, - CAST( - ARRAY( - SELECT pg_get_indexdef(idx.indexrelid, k + 1, true) - FROM generate_subscripts(idx.indkey, 1) as k - ORDER BY k - ) as varchar - ) as columns -FROM pg_index as idx -JOIN pg_class as i -ON i.oid = idx.indexrelid -JOIN pg_am as am -ON i.relam = am.oid -JOIN pg_namespace as ns -ON ns.oid = i.relnamespace -AND ns.nspname = ANY(current_schemas(false)) -ORDER BY idx.indrelid -" -} diff --git a/utilities/dbviz/src/schema.rs b/utilities/dbviz/src/schema.rs deleted file mode 100644 index da07eb9b5..000000000 --- a/utilities/dbviz/src/schema.rs +++ /dev/null @@ -1,84 +0,0 @@ -//! Core entities. -use std::collections::HashMap; - -use serde::{Deserialize, Serialize}; - -/// All the schema information. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct Schema { - /// List of tables in the database. - pub(crate) tables: Vec, - /// List of relations in the database. - pub(crate) relations: Vec, - /// Partial Tables - pub(crate) partial_tables: HashMap>, -} - -/// Table information. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct TableColumn { - /// Column name. - pub(crate) column: String, - /// Column data type. - pub(crate) data_type: String, - /// Column index. - pub(crate) index: i32, - /// Column default. - pub(crate) default: Option, - /// Column nullable. - pub(crate) nullable: String, - /// Column max chars. - pub(crate) max_chars: Option, - /// Column description. - pub(crate) description: Option, - /// Table description. - pub(crate) table_description: Option, // Redundant but easiest way to get it. - /// Column primary key. - pub(crate) primary_key: bool, -} - -/// Table information. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct Table { - /// Table name. - pub(crate) name: String, - /// Table Description - pub(crate) description: Option, - /// List of fields. - pub(crate) fields: Vec, -} - -/// Row description. -//#[derive(Debug)] -// pub(crate)struct Field(pub(crate)FieldName, pub(crate)FieldType); - -/// Relation node. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct Relation { - /// Table that the constraint references. - pub(crate) on_table: TableName, - /// Field that the constraint references. - pub(crate) on_field: FieldName, - /// Table which the fk references. - pub(crate) to_table: TableName, - /// Field which the fk references. - pub(crate) to_field: FieldName, -} - -/// Table name -pub(crate) type TableName = String; -/// Field name -pub(crate) type FieldName = String; -// pub(crate)type FieldType = String; - -/// Index Definition -pub(crate) struct Index { - /// Table name - pub(crate) table: TableName, - // pub(crate)name: String, - /// Primary Key - pub(crate) primary: bool, - // pub(crate)unique: bool, - /// Fields - pub(crate) fields: Vec, -}