diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 33cf544..aced2f1 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -21,9 +21,6 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true -defaults: - run: - working-directory: ./optd-persistent name: check jobs: fmt: @@ -61,22 +58,6 @@ jobs: components: clippy - name: cargo clippy run: cargo clippy --locked --all-targets --all-features -- -D warnings - doc: - # run docs generation on nightly rather than stable. This enables features like - # https://doc.rust-lang.org/beta/unstable-book/language-features/doc-cfg.html which allows an - # API be documented as only available in some specific platforms. - runs-on: ubuntu-latest - name: nightly / doc - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install nightly - uses: dtolnay/rust-toolchain@nightly - - name: Install cargo-docs-rs - uses: dtolnay/install@cargo-docs-rs - - name: cargo docs-rs - run: cargo docs-rs hack: # cargo-hack checks combinations of feature flags to ensure that features are all additive # which is required for feature unification diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d6253a0..565c9eb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,9 +17,6 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true -defaults: - run: - working-directory: ./optd-persistent name: test jobs: required: diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..86281ff --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +# Contains compiled files and executables. +debug/ +target/ + +# These are backup files generated by rustfmt. +**/*.rs.bk + +# Ignore any database files. +**/*.db + +# We will check in all code-generated entity files, as newer versions of `sea-orm-cli` might +# conflict with previous versions. +# **/entities diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..8acb13b --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2846 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aa100a6f6f525226719f8de3f70076be4f4191801ebd92621450d1c51e9053d" + +[[package]] +name = "ahash" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + +[[package]] +name = "allocator-api2" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52f4a9cf8f3ff707b4eb1acd0136efd8b3bec6b345ed32fcab47c0a5c99b800" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e579a7752471abc2a8268df8b20005e3eadd975f585398f17efcfd8d4927371" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is-terminal", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" + +[[package]] +name = "anstyle-parse" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "anstyle-wincon" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcd8291a340dd8ac70e18878bc4501dd7b4ff970cfa21c207d36ece51ea88fd" +dependencies = [ + "anstyle", + "windows-sys 0.48.0", +] + +[[package]] +name = "arrayvec" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a2f58b0bb10c380af2b26e57212856b8c9a59e0925b4c20f4a174a49734eaf7" + +[[package]] +name = "async-stream" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a26cb53174ddd320edfff199a853f93d571f48eeb4dde75e67a9a3dbb7b7e5e" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db134ba52475c060f3329a8ef0f8786d6b872ed01515d4b79c162e5798da1340" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.98", +] + +[[package]] +name = "async-trait" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6287685011f026b98d26afd53251ad0101e856531b423eb2384265f7d4f5b01" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.98", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "autocfg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" + +[[package]] +name = "backtrace" +version = "0.3.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88fb5a785d6b44fd9d6700935608639af1b8356de1e55d5f7c2740f4faa15d82" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + +[[package]] +name = "base64" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" + +[[package]] +name = "base64ct" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71acf5509fc522cce1b100ac0121c635129bfd4d91cdf036bcc9b9935f97ccf5" + +[[package]] +name = "bigdecimal" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5274a6b6e0ee020148397245b973e30163b7bffbc6d473613f850cb99888581e" +dependencies = [ + "libm", + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "bitflags" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d155346769a6855b86399e9bc3814ab343cd3d62c7e985113d46a0ec3c281fd" + +[[package]] +name = "bitflags" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +dependencies = [ + "serde", +] + +[[package]] +name = "block-buffer" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03588e54c62ae6d763e2a80090d50353b785795361b4ff5b3bf0a5097fc31c0b" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12ae9db68ad7fac5fe51304d20f016c911539251075a214f8e663babefa35187" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" + +[[package]] +name = "cc" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "serde", + "windows-targets 0.52.0", +] + +[[package]] +name = "clap" +version = "4.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384e169cc618c613d5e3ca6404dda77a8685a63e08660dcc64abaf7da7cb0c7a" +dependencies = [ + "clap_builder", + "clap_derive", + "once_cell", +] + +[[package]] +name = "clap_builder" +version = "4.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef137bbe35aab78bdb468ccfba75a5f4d8321ae011d34063770780545176af2d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "clap_lex" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" + +[[package]] +name = "codespan-reporting" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6ce42b8998a383572e0a802d859b1f00c79b7b7474e62fff88ee5c2845d9c13" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + +[[package]] +name = "concurrent-queue" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520fbf3c07483f94e3e3ca9d0cfd913d7718ef2483d2cfd91c0d9e91474ab913" + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "cpufeatures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53757d12b596c16c78b83458d732a5d1a17ab3f53f2f7412f6fb57cc8a140ab3" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d0165d2900ae6778e36e80bbc4da3b5eefccee9ba939761f9c2882a5d9af3ff" + +[[package]] +name = "crossbeam-queue" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b10ddc024425c88c2ad148c1b0fd53f4c6d38db9697c9f1588381212fa657c9" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "crypto-common" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "cxx" +version = "1.0.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c53d75fe543215ca091d792e13351dcb940842dd2829b2a2dd43ab4bd1a015" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "618f85c8f132bd8912aab124e15a38adc762bb7e3cef84524adde1692ef3e8bc" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn 1.0.98", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca21461be76a23df4f63a2107a0bb406ef41548e635ff7edcbd1ab5a6bb997e2" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8da0a2c0697647b5824844a5d2dedcd97a2d7b75e6e4d0b8dd183e4081e1cf" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.98", +] + +[[package]] +name = "darling" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c99d16b88c92aef47e58dadd53e87b4bd234c29934947a6cec8b466300f99b" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ea05d2fcb27b53f7a98faddaf5f2914760330ab7703adfc9df13332b42189f9" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "darling_macro" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bfb82b62b1b8a2a9808fb4caf844ede819a76cfc23b2827d7f94eefb49551eb" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "der" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19c5cb402c5c958281c7c0702edea7b780d03b86b606497ca3a10fcd3fc393ac" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" +dependencies = [ + "powerfmt", + "serde", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d86534ed367a67548dc68113a0f5db55432fdfbb6e6f9d77704397d95d5780" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "dotenvy" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9155c8f4dc55c7470ae9da3f63c6785245093b3f6aeb0f5bf2e968efbba314" +dependencies = [ + "dirs", +] + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +dependencies = [ + "serde", +] + +[[package]] +name = "equivalent" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1" + +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b5fb89194fa3cad959b833185b3063ba881dbfc7030680b314250779fb4cc91" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "flume" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +dependencies = [ + "futures-core", + "futures-sink", + "spin 0.9.8", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece68d15c92e84fa4f19d3780f1294e5ca82a78a6d515f1efaabcc144688be00" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fa4cc29d25b0687b8570b0da86eac698dcb525110ad8b938fe6712baa711ec" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29d6d2ff5bb10fb95c85b8ce46538a2e5f5e7fdc755623a7d4529ab8a4ed9d2a" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashlink" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692eaaf7f7607518dd3cef090f1474b61edc5301d8012f09579920df68b725ee" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f41e9c77b6fc05b57497b960aad55942a9bbc5b20e1e623cf7fb1868f695d1" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddca131f3e7f2ce2df364b57949a9d47915cfbd35e46cfee355ccebbf794d6a2" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c17cc76786e99f8d2f055c11159e7f0091c42474dcc3189fbab96072e873e6d" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "inherent" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c05a410d53e44fc943a35a32ca27e32af2ea004d5107ccef685d022fc2b9fb" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.98", +] + +[[package]] +name = "is-terminal" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24fddda5af7e54bf7da53067d6e802dbcc381d0a8eef629df528e3ebf68755cb" +dependencies = [ + "hermit-abi", + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "itertools" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92a9df60778f789c37f76778ae8d0a2471c41baa8b059d98a5873c978f549587" + +[[package]] +name = "itoa" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" + +[[package]] +name = "js-sys" +version = "0.3.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d99f9e3e84b8f67f846ef5b4cbbc3b1c29f6c759fcbce6f01aa0e73d932a24c" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a374c89b9db55895453a74c1e38861d9deec0b01b405a82516e9d5de4820dea1" +dependencies = [ + "spin 0.4.10", +] + +[[package]] +name = "libc" +version = "0.2.152" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" + +[[package]] +name = "libm" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "link-cplusplus" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dfb9f65d9966f6ca6522043978030b564f3291af987fbf1dd55b6a064ba1b36" +dependencies = [ + "cc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" + +[[package]] +name = "lock_api" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" + +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matches" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15305656809ce5a4805b1ff2946892810992197ce1270ff79baded852187942e" + +[[package]] +name = "md-5" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6a38fc55c8bbc10058782919516f88826e70320db6d206aebc49611d24216ae" +dependencies = [ + "digest", +] + +[[package]] +name = "memchr" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" + +[[package]] +name = "minimal-lexical" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6595bb28ed34f43c3fe088e48f6cfb2e033cab45f25a5384d5fdf564fbc8c4b2" + +[[package]] +name = "miniz_oxide" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be0f75932c1f6cfae3c04000e40114adf955636e19040f9c0a2c380702aa1c7f" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.45.0", +] + +[[package]] +name = "nom" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffd9d26838a953b4af82cbeb9f1592c6798916983959be223a7124e992742c1" +dependencies = [ + "memchr", + "minimal-lexical", + "version_check", +] + +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2399c9463abc5f909349d8aa9ba080e0b88b3ce2885389b60b993f39b1a56905" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-integer" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af3fdbbc3291a5464dc57b03860ec37ca6bf915ed6ee385e7c6c052c422b2124" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51a3322e4bca9d212ad9a158a02abc6934d005490c054a2778df73a70aa0a30" +dependencies = [ + "libc", +] + +[[package]] +name = "object" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4" + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "optd-mvp" +version = "0.1.0" +dependencies = [ + "async-stream", + "async-trait", + "fxhash", + "sea-orm", + "sea-orm-migration", + "serde", + "serde_json", + "strum", + "thiserror 2.0.0", + "tokio", + "trait-variant", +] + +[[package]] +name = "ordered-float" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d84eb1409416d254e4a9c8fa56cc24701755025b458f0fcd8e59e1f5f40c23bf" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ouroboros" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c86de06555b970aec45229b27291b53154f21a5743a163419f4e4c0b065dcde" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3633d65683f13b9bcfaa3150880b018899fb0e5d0542f4adaea4f503fdb5eabf" +dependencies = [ + "heck 0.4.1", + "itertools 0.12.0", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + +[[package]] +name = "parking_lot" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2f4f894f3865f6c0e02810fc597300f34dc2510f66400da262d8ae10e75767d" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys 0.29.0", +] + +[[package]] +name = "paste" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0744126afe1a6dd7f394cb50a716dbe086cb06e255e53d8d0185d82828358fb5" + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pin-project-lite" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea" + +[[package]] +name = "proc-macro-error" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d259aa4825fa1a2371419d30a520219feff9fb3591550a209b4477d2ebaae4f" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.98", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd21889899aa8e1ca2b924c1d3f08086631fc90768225b3268b5d5c3e806a503" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.98", + "syn-mid", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b33eb56c327dec362a9e55b3ad14f9d2f0904fb5a5b03b513ab5465399e9f43" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "606c4ba35817e2922a308af55ad51bab3645b59eae5c570d4a6cf07e36bd493b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "version_check", + "yansi", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redox_syscall" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc" +dependencies = [ + "bitflags 1.1.0", +] + +[[package]] +name = "redox_users" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" +dependencies = [ + "getrandom", + "redox_syscall", +] + +[[package]] +name = "regex" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72457500f2cf948feb4efccaeb460570c8f66ee5ba33c936bb4bfaa628d71853" +dependencies = [ + "byteorder", + "regex-syntax", + "utf8-ranges", +] + +[[package]] +name = "regex-syntax" +version = "0.6.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" + +[[package]] +name = "ring" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb9d44f9bf6b635117787f72416783eb7e4227aaf255e5ce739563d817176a7e" +dependencies = [ + "cc", + "getrandom", + "libc", + "spin 0.9.8", + "untrusted", + "windows-sys 0.48.0", +] + +[[package]] +name = "rsa" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dd2017d3e6d67384f301f8b06fbf4567afc576430a61624d845eb04d2b30a72" +dependencies = [ + "byteorder", + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-iter", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "subtle", + "zeroize", +] + +[[package]] +name = "rust_decimal" +version = "1.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee9164faf726e4f3ece4978b25ca877ddc6802fa77f38cdccb32c7f805ecd70c" +dependencies = [ + "arrayvec", + "num-traits", + "serde", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3058a43ada2c2d0b92b3ae38007a2d0fa5e9db971be260e0171408a4ff471c95" + +[[package]] +name = "rustix" +version = "0.38.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +dependencies = [ + "bitflags 2.4.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4828ea528154ae444e5a642dbb7d5623354030dc9822b83fd9bb79683c7399d0" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pemfile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" +dependencies = [ + "base64 0.21.0", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" + +[[package]] +name = "rustls-webpki" +version = "0.102.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "ryu" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92464b447c0ee8c4fb3824ecc8383b81717b9f1e74ba2e72540aef7b9f82997" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "scratch" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e114536316b51a5aa7a0e59fc49661fd263c5507dd08bd28de052e57626ce69" + +[[package]] +name = "sea-bae" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bd3534a9978d0aa7edd2808dc1f8f31c4d0ecd31ddf71d997b3c98e9f3c9114" +dependencies = [ + "heck 0.4.1", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "sea-orm" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5680a8b686985116607ef5f5af2b1f9e1cc2c228330e93101816a0baa279afa" +dependencies = [ + "async-stream", + "async-trait", + "bigdecimal", + "chrono", + "futures", + "log", + "ouroboros", + "rust_decimal", + "sea-orm-macros", + "sea-query", + "sea-query-binder", + "serde", + "serde_json", + "sqlx", + "strum", + "thiserror 1.0.35", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "sea-orm-cli" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aefbd960c9ed7b2dfbab97b11890f5d8c314ad6e2f68c7b36c73ea0967fcc25" +dependencies = [ + "chrono", + "clap", + "dotenvy", + "glob", + "regex", + "sea-schema", + "tracing", + "tracing-subscriber", + "url", +] + +[[package]] +name = "sea-orm-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a239e3bb1b566ad4ec2654d0d193d6ceddfd733487edc9c21a64d214c773910" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "sea-bae", + "syn 2.0.87", + "unicode-ident", +] + +[[package]] +name = "sea-orm-migration" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa7bbfbe3bec60b5925193acc9c98b9f8ae9853f52c8004df0c1ea5193c01ea0" +dependencies = [ + "async-trait", + "clap", + "dotenvy", + "futures", + "sea-orm", + "sea-orm-cli", + "sea-schema", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "sea-query" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff504d13b5e4b52fffcf2fb203d0352a5722fa5151696db768933e41e1e591bb" +dependencies = [ + "bigdecimal", + "chrono", + "inherent", + "ordered-float", + "rust_decimal", + "sea-query-derive", + "serde_json", + "time", + "uuid", +] + +[[package]] +name = "sea-query-binder" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0019f47430f7995af63deda77e238c17323359af241233ec768aba1faea7608" +dependencies = [ + "bigdecimal", + "chrono", + "rust_decimal", + "sea-query", + "serde_json", + "sqlx", + "time", + "uuid", +] + +[[package]] +name = "sea-query-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9834af2c4bd8c5162f00c89f1701fb6886119a88062cf76fe842ea9e232b9839" +dependencies = [ + "darling", + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 2.0.87", + "thiserror 1.0.35", +] + +[[package]] +name = "sea-schema" +version = "0.16.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2a4ff9e87c4340affbec4f7790d724dcd87e71fcd0ffe2247481843380485aa" +dependencies = [ + "futures", + "sea-query", + "sea-schema-derive", +] + +[[package]] +name = "sea-schema-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "debdc8729c37fdbf88472f97fd470393089f997a909e535ff67c544d18cfccf0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "serde" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b114498256798c94a0689e1a15fec6005dee8ac1f41de56404b67afc2a4b773" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3385e45322e8f9931410f01b3031ec534c3947d0e94c18049af4d9f9907d4e0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "serde_json" +version = "1.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d947f6b3163d8857ea16c4fa0dd4840d52f3041039a85decd46867eb1abef2e4" +dependencies = [ + "itoa 1.0.1", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" +dependencies = [ + "form_urlencoded", + "itoa 0.4.0", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c77f4e7f65455545c2153c1253d25056825e77ee2533f0e41deb65a93a34852f" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99c3bd8169c58782adad9290a9af5939994036b76187f7b4f0e6de91dbbfc0ec" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signature" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fe458c98333f9c8152221191a77e2a44e8325d0193484af2e9421a53019e57d" +dependencies = [ + "digest", + "rand_core", +] + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +dependencies = [ + "serde", +] + +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "spin" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ceac490aa12c567115b40b7b7fceca03a6c9d53d5defea066123debc83c5dc1f" + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a5be806ab6f127c3da44b7378837ebf01dadca8510a0e572460216b228bd0e" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlformat" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87e292b4291f154971a43c3774364e2cbcaec599d3f5bf6fa9d122885dbc38a" +dependencies = [ + "itertools 0.10.0", + "nom", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93334716a037193fac19df402f8571269c84a00852f6a7066b5d2616dcd64d3e" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d8060b456358185f7d50c55d9b5066ad956956fddec42ee2e8567134a8936e" +dependencies = [ + "atoi", + "bigdecimal", + "byteorder", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown", + "hashlink", + "hex", + "indexmap", + "log", + "memchr", + "once_cell", + "paste", + "percent-encoding", + "rust_decimal", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlformat", + "thiserror 1.0.35", + "time", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid", + "webpki-roots", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cac0692bcc9de3b073e8d747391827297e075c7710ff6276d9f7a1f3d58c6657" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.87", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1804e8a7c7865599c9c79be146dc8a9fd8cc86935fa641d3ea58e5f0688abaa5" +dependencies = [ + "dotenvy", + "either", + "heck 0.5.0", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.87", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64bb4714269afa44aef2755150a0fc19d756fb580a67db8885608cf02f47d06a" +dependencies = [ + "atoi", + "base64 0.22.0", + "bigdecimal", + "bitflags 2.4.0", + "byteorder", + "bytes", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa 1.0.1", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa", + "rust_decimal", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 1.0.35", + "time", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fa91a732d854c5d7726349bb4bb879bb9478993ceb764247660aee25f67c2f8" +dependencies = [ + "atoi", + "base64 0.22.0", + "bigdecimal", + "bitflags 2.4.0", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa 1.0.1", + "log", + "md-5", + "memchr", + "num-bigint", + "once_cell", + "rand", + "rust_decimal", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 1.0.35", + "time", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5b2cf34a45953bfd3daaf3db0f7a7878ab9b7a6b91b422d24a7a9e4c857b680" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "stringprep" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "strum" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117c413ac8a6cc19c773939932477a341e416eff7f0e84db42f091d85d7c6e0e" + +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + +[[package]] +name = "syn" +version = "1.0.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn-mid" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.98", +] + +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if", + "fastrand", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "termcolor" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a52c023823933499250b43960b272e25336c6e2ab8684672edc34489f049ccdd" +dependencies = [ + "wincolor", +] + +[[package]] +name = "thiserror" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c53f98874615aea268107765aa1ed8f6116782501d18e53d08b471733bea6c85" +dependencies = [ + "thiserror-impl 1.0.35", +] + +[[package]] +name = "thiserror" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15291287e9bff1bc6f9ff3409ed9af665bec7a5fc8ac079ea96be07bca0e2668" +dependencies = [ + "thiserror-impl 2.0.0", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8b463991b4eab2d801e724172285ec4195c650e8ec79b149e6c2a8e6dd3f783" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.98", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22efd00f33f93fa62848a7cab956c3d38c8d43095efda1decfc2b3a5dc0b8972" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "thread_local" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +dependencies = [ + "once_cell", +] + +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa 1.0.1", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tokio" +version = "1.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d3ce25f50619af8b0aec2eb23deebe84249e19e2ddd393a6e16e3300a6dadfd" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-macros" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "tokio-stream" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.98", +] + +[[package]] +name = "tracing-core" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" +dependencies = [ + "matchers", + "once_cell", + "regex", + "sharded-slab", + "thread_local", + "tracing", + "tracing-core", +] + +[[package]] +name = "trait-variant" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "typenum" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" + +[[package]] +name = "unicode-bidi" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2560b941fdb9ea38301b9b708504d612fcdf9c91a8c31d82219bd74cb07d304d" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-ident" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee" + +[[package]] +name = "unicode-normalization" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51ccda9ef9efa3f7ef5d91e8f9b83bbe6955f9bf86aec89d5cce2c874625920f" + +[[package]] +name = "unicode-width" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc85732b6d55a0d520aaf765536a188d9d993770c28633422f85bb646da61335" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + +[[package]] +name = "utf8-ranges" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122" + +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + +[[package]] +name = "uuid" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd6469f4314d5f1ffec476e05f17cc9a78bc7a27a6a857842170bdf8d6f98d2f" +dependencies = [ + "serde", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83240549659d187488f91f33c0f8547cbfef0b2088bc470c116d1d260ef623d9" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae70622411ca953215ca6d06d3ebeb1e915f0f6613e3b495122878d7ebec7dae" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn 1.0.98", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e734d91443f177bfdb41969de821e15c516931c3c3db3d318fa1b68975d0f6f" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53739ff08c8a68b0fdbcd54c372b8ab800b1449ab3c9d706503bc7dd1621b2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.98", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9a543ae66aa233d14bb765ed9af4a33e81b8b58d1584cf1b47ff8cd0b9e4489" + +[[package]] +name = "web-sys" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "721c6263e2c66fd44501cc5efbfa2b7dfa775d13e4ea38c46299646ed1f9c70a" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de2cfda980f21be5a7ed2eadb3e6fe074d56022bea2cdeb1a62eb220fc04188" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "whoami" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524b58fa5a20a2fb3014dd6358b70e6579692a56ef6fce928834e488f42f65e8" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "winapi" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ad91d846a4a5342c1fb7008d26124ee6cf94a3953751618577295373b32117" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a16a8e2ebfc883e2b1771c6482b1fb3c6831eab289ba391619a2d93a7356220f" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ca29cb03c8ceaf20f8224a18a530938305e9872b1478ea24ff44b4f503a1d1d" + +[[package]] +name = "wincolor" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9dc3aa9dcda98b5a16150c54619c1ead22e3d3a5d458778ae914be760aa981a" +dependencies = [ + "winapi", +] + +[[package]] +name = "windows" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ceb069ac8b2117d36924190469735767f0990833935ab430155e71a44bafe148" +dependencies = [ + "windows_aarch64_msvc 0.29.0", + "windows_i686_gnu 0.29.0", + "windows_i686_msvc 0.29.0", + "windows_x86_64_gnu 0.29.0", + "windows_x86_64_msvc 0.29.0", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d027175d00b01e0cbeb97d6ab6ebe03b12330a35786cbaca5252b1c4bf5d9b" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + +[[package]] +name = "windows_i686_gnu" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8793f59f7b8e8b01eda1a652b2697d87b93097198ae85f823b969ca5b89bba58" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + +[[package]] +name = "windows_i686_msvc" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8602f6c418b67024be2996c512f5f995de3ba417f4c75af68401ab8756796ae4" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d615f419543e0bd7d2b3323af0d86ff19cbc4f816e6453f36a2c2ce889c354" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d95421d9ed3672c280884da53201a5c46b7b2765ca6faf34b0d71cf34a3561" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + +[[package]] +name = "yansi" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fc79f4a1e39857fc00c3f662cbf2651c771f00e9c15fe2abc341806bd46bd71" + +[[package]] +name = "zerocopy" +version = "0.7.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c4061bedbb353041c12f413700357bec76df2c7e2ca8e4df8bac24c6bf68e3d" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "zeroize" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" diff --git a/optd-mvp/Cargo.toml b/optd-mvp/Cargo.toml new file mode 100644 index 0000000..f4a3a62 --- /dev/null +++ b/optd-mvp/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "optd-mvp" +version = "0.1.0" +edition = "2021" +authors = ["Connor Tsui"] + +[dependencies] + +# `VARIANTS` constant made public in "1.1.1". +sea-orm = { version = "1.1.1", features = [ + "sqlx-sqlite", + "runtime-tokio-rustls", + "macros", + "debug-print", + "with-json", +] } +sea-orm-migration = "1.0" +serde = "1.0" +serde_json = "1.0.118" # Support `Hash` on `serde_json::Value` in "1.0.118". +tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] } +trait-variant = "0.1.2" # Support `make(Send)` syntax in "0.1.2". +thiserror = "2.0" +fxhash = "0.2" + +# Pin more recent versions for `-Zminimal-versions`. +async-trait = "0.1.43" # Remove lifetime parameter from "0.1.42". +async-stream = "0.3.1" # Fix unsatisfied trait bound from "0.3.0". +strum = "0.26.0" # Fix `std::marker::Sized` from "0.25.0". + diff --git a/optd-mvp/DESIGN.md b/optd-mvp/DESIGN.md new file mode 100644 index 0000000..6d1863d --- /dev/null +++ b/optd-mvp/DESIGN.md @@ -0,0 +1,214 @@ +# Duplicate Elimination Memo Table + +_Connor Tsui, December 2024_ + +Note that most of the details are in `src/memo/persistent/implementation.rs`. + +For this document, we are assuming that the memo table is backed by a database / ORM. Both the +problems and the features detailed in this document are unique to this design, and likely do not +apply to an in-memory memo table. + +## Group Merging + +During logical exploration, there will be rules that create cycles between groups. The easy solution +for this is to immediately merge two groups together when the engine determines that adding an +expression would result in a duplicate expression from another group. + +However, if we want to support parallel exploration, this could be prone to high contention. By +definition, merging group 1 into group 2 would mean that _every expression_ that has a child of +group 1 with would need to be rewritten to point to group 2 instead. + +This is prohibitive in a parallel setting, as that would mean every single task that gets affected +would need to either wait for the rewrites to happen before resuming work, or potentially need to +abort their work because data has changed underneath them. + +So immediate / eager group merging is not a great idea for parallel exploration. However, if we +don't merge two groups that are equivalent, we are subject to doing duplicate work for every +duplicate expression in the memo table during physical optimization. + +Instead of merging groups together immediately, we can instead maintain an auxiliary data structure +that records the groups that _eventually_ need to get merged, and "lazily" merge those groups +together once every group has finished exploration. We will refer to merging groups as the act of +recording that the groups should eventually be merged together after exploration is finished. + +## Union-Find Group Sets + +We use the well-known Union-Find algorithm and corresponding data structure as the auxiliary data +structure that tracks the to-be-merged groups. + +Union-Find supports `Union` and `Find` operations, where `Union` merges sets and `Find` searches for +a "canonical" or "root" element that is shared between all elements in a given set. Note that we +will also support an iteration operation that iterates over all elements in a given set. We will +need this for [duplicate detection](#fingerprinting--group-merge), which is explained below. + +For more information about Union-Find, see these +[15-451 lecture notes](https://www.cs.cmu.edu/~15451-f24/lectures/lecture08-union-find.pdf). We will +use the exact same data structure, but add an additional `next` pointer for each node that embeds +a circular linked list for each set. + +Here, we make the elements the groups themselves (really the group IDs), which allows us to merge +group sets together and also determine a "root group" that all groups in a set can agree on. + +When every group in a group set has finished exploration, we can safely begin to merge them +together by moving all expressions from every group in the group set into a single large group. +Other than making sure that any reference to an old group in the group set points to this new large +group, exploration of all groups is done and physical optimization can start. + +Note that since we are now waiting for exploration of all groups to finish, this algorithm is much +closer to the Volcano framework than the Cascades' incremental search. However, since we eventually +will want to store trails / breadcrumbs of decisions made to skip work in the future, and since we +essentially have unlimited space due to the memo table being backed by a DBMS, this is not as much +of a problem. + +## Duplicate Detection + +Deciding that we will merge groups lazily does not solve all of our problems. We have to know _when_ +we want to merge these groups. + +A naive approach is to simply loop over every expression in the memo table and check if we are about +to insert a duplicate. This, of course, is bad for performance. + +We will use a fingerprinting / hashing method to detect when a duplicate expression might be +inserted into the memo table (returning an error instead of inserting), and we will use that to +trigger group merges. + +For every logical expression we insert into the memo table, we will create a fingerprint that +contains both the kind of expression / relation (Scan, Filter, Join) and a hash of all +information that makes that expression unique. For example: + +- The fingerprint of a Scan should probably contain a hash of the table name and the pushdown + predicate. +- The fingerprint of a Filter should probably contain a hash of its child group ID and predicate. +- The fingerprint of a Join should probably contain a hash of the left group ID and the right group + ID, as well as the join predicate. + +Note that the above descriptions are slightly inaccurate, and we'll explain why in a later +[section](#fingerprinting--group-merge). + +Also, if we have duplicate detection for logical expression, and we do not start physical +optimization until after full plan enumeration, then we do not actually need to do duplicate +detection of physical expressions, since they are derivative of the deduplicated logical +expressions. + +### Fingerprint Matching Algorithm + +When an expression is added to the memo table, it will first calculate the fingerprint of the +expression. The memo table will compare this fingerprint with every fingerprint in the memo table to +check if we have seen this expression before (in any group). While this is effectively a scan +through every expression, supporting the fingerprint table with an B+tree index will speed up this +operation dramatically (since these fingerprints can be sorted by expression / relation kind). + +If there are no identical fingerprints, then there is no duplicate expression, and we can safely +add the expression into the memo table. However, if there are matching fingerprints, we need to +further check for false positives due to hash collisions. + +We do full exact match equality checks with every expression that had a fingerprint match. If there +are no exact matches, then we can safely add the expression into the memo table. However, if we find +an exact match (note that there can be at most one exact match since we have an invariant that there +cannot be duplicate expressions), then we know that the expression we are trying to add already +exists in the memo table. + +### Fingerprinting + Group Merge + +There is a slight problem with the algorithm described above. It does not account for when a child +group has merged into another group. + +For example, let's say we have groups 1, 2, and 3. We insert an expression Join(1, 2) into the +memo table with its fingerprint calculated with groups 1 and 2. It is possible that we find out that +groups 2 and 3 need to merged. This means that Join(1, 2) and Join (1, 3) are actually identical +expressions, and the fingerprinting strategies for expressions described above do not handle this. + +We will solve this problem by adding allowing multiple fingerprints to reference the same logical +expression, and we will generate a new fingerprint for every expression that is affected by a group +merge / every expression who's parent group now has a new root group. + +In the above scenario, we will find every expression in the memo table that has group 2 as a child. +For each expression, we will generate another fingerprint with group 2 "rewritten" as group 3 in the +hash. Note that we _do not_ modify the original expression, we are simply adding another fingerprint +into the memo table. + +Finally, we need to handle when multiple groups in a group set are merged into another group set. +For example, if a left group set { 1, 2, 3, 4, 5 } with root 1 needs to be merged into a right group +set { 6, 7, 8, 9, 10 } with root 6, then we need to generate a new fingerprint for every expression +in groups 1, 2, 3, 4, and 5 with group 1 "rewritten" as group 6. + +More formally, we are maintaining this invariant: +**For every expression, there exists a fingerprint that maps back to the expression that uses the** +**root groups of their children to calculate the hash.** + +For example, if we have a group set { 1, 3, 5 } with root group 1 and group set { 2, 4, 6 } with +root group 2, the fingerprint of Join(5, 4) should really be a fingerprint of Join(1, 2). + +This invariant means that when we are checking if some expression already exists, we should use the +root groups of the child groups in our expression to calculate the fingerprint, and we can guarantee +that no fingerprint matches implies no duplicates. + +A further implication of this invariant means that new fingerprints need to be generated every time +we merge groups. If we have a left group set { 1, 3, 5 } with root group 1 and right group set +{ 2, 4, 6 } with root group 2, and we merge the first group set into the second, then every +expression that has a child group of 1, 3, or 5 now has a stale fingerprint that uses root group 1 +instead of root group 2. + +Thus, when we merge the left group into the right group, we need to do the following: + +1. Gather the group set, i.e. every single group that has root group 1 (iterate) +2. Retrieve every single expression that has a child group in the group set (via junction table) +3. Generate a new fingerprint for each expression and add it into the memo table + +The speed of steps 2 and 3 above are largely dependent on the backing DBMS. However, we can support +step 1 directly in the union find data structure by maintain a circular linked list for every set. +Each group now tracks both a `parent` pointer and a `next` pointer. When merging / unioning a set +into another set, we swap the `next` pointers of the two roots to maintain the circular linked list. +This allows us to do step 1 in linear time relative to the size of the group set. + +### Discovered Duplicates + +The above algorithm has one more problem: merging groups can cause the memo table to "discover" that +there are duplicate expressions in the memo table. + +Here is an example: let's say we have the following groups, each with one expression (note that the +example will work even with multiple expressions): + +1. `Scan(1)` +2. `Scan(2)` +3. `Filter(1)` +4. `Filter(2)` +5. `Filter(4)` +6. `Join(3, 4)` +7. `Join(3, 5)` +8. `Sort(6)` +9. `Sort(7)` + +Note how groups 5 is just a second filter on top of group 2. Suppose that we find out that +`(Filter(4) = Filter(Filter(2))) == Filter(2)`. In that case, we need to merge groups 4 and 5. The +problem here is that groups 6 and 7 are considered separate groups, but we have now discovered that +they are actually the same. The same is true for groups 8 and 9. In this scenario, the merging of +groups has "generated" a duplicate expression. + +However, this is not as big of a problem as it might seem. The issue we want to avoid is lots of +duplicate work or even an infinite loop of rule application. Observe that if we apply a rule to both +the expression in group 6 and group 7 that we will get the same exact expression. + +For example, if we apply join commutativity to the expression in group 6 (`Join(3, 4)`), we would +add `Join(4, 3)` into group 6. When we apply join commutativity to the expression in group 7 +(`Join(3, 5)`), we would get back `Join(5, 3)`. However, the memo table will detect this as a +duplicate because it will use the root group of 4 and 5 to generate the fingerprint and see that +`Join(4, 3)` already exists. Again, similar logic applies for groups 8 and 9. + +At a high level, almost all of our operations are lazy. Work does not need to be done unless it is +absolutely necessary for correctness. By allowing some amount of duplicates, we get some nice +properties with respect to parallelizing memo table access. + +## Efficiency and Parallelism + +Fingerprinting by itself is very efficient, as creating a fingerprint and looking up a fingerprint +can be made quite efficient with indexes. The real concern here is that merging two groups is very, +very expensive. Depending on the workload, it is both possible that the amortized cost is low or +that group merging takes a majority of the work. + +However, we must remember that we want to parallelize access to the memo table. The above algorithms +are notably **read and append only**. There is never a point where we need to update an expression +to maintain invariants. This is important, as it means that we can add and lookup expression and +groups _without having to take any locks_. If we enforce a serializable isolation level, then every +method on the memo table can be done in parallel with relatively low contention due to there being +zero write-write conflicts. diff --git a/optd-mvp/entities.md b/optd-mvp/entities.md new file mode 100644 index 0000000..fc13a39 --- /dev/null +++ b/optd-mvp/entities.md @@ -0,0 +1,27 @@ +# Generate the `entities` module + +To make changes to the database tables and schema, you will have to modify files in the `migrator` module and then update the `entities` module using `sea-orm-cli`. + +This assumes that you already have the `sqlite3` binary installed. First, make sure you have installed `sea-orm-cli`: + +```sh +$ cargo install sea-orm-cli +``` + +Make sure your working directory is in the crate root (not workspace): + +```sh +$ cd optd-mvp +``` + +If you have not generate the `sqlite.db` file yet, you will need to run this command which will generate the `sqlite.db` file and run all of the migrations: + +```sh +$ cargo run --bin migrate +``` + +Finally, run this command to generate / overwrite the `entities` module in the `src` directory. + +```sh +$ sea-orm-cli generate entity -u sqlite:./sqlite.db -o src/entities +``` diff --git a/optd-mvp/src/bin/migrate.rs b/optd-mvp/src/bin/migrate.rs new file mode 100644 index 0000000..0634f23 --- /dev/null +++ b/optd-mvp/src/bin/migrate.rs @@ -0,0 +1,19 @@ +//! A simple script that generates the database file needed for `sea-orm-cli` to extract the schemas +//! from and generate the `entities` module. + +use optd_mvp::{migrate, DATABASE_FILENAME, DATABASE_URL}; +use sea_orm::*; +use sea_orm_migration::prelude::*; + +#[tokio::main] +async fn main() { + let _ = std::fs::remove_file(DATABASE_FILENAME); + + let db = Database::connect(DATABASE_URL) + .await + .expect("Unable to connect to the database"); + + migrate(&db) + .await + .expect("Something went wrong during migration"); +} diff --git a/optd-mvp/src/entities/fingerprint.rs b/optd-mvp/src/entities/fingerprint.rs new file mode 100644 index 0000000..608ca57 --- /dev/null +++ b/optd-mvp/src/entities/fingerprint.rs @@ -0,0 +1,33 @@ +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "fingerprint")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i32, + pub logical_expression_id: i32, + pub kind: i16, + pub hash: i64, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::logical_expression::Entity", + from = "Column::LogicalExpressionId", + to = "super::logical_expression::Column::Id", + on_update = "Cascade", + on_delete = "Cascade" + )] + LogicalExpression, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::LogicalExpression.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/optd-mvp/src/entities/group.rs b/optd-mvp/src/entities/group.rs new file mode 100644 index 0000000..b5b1686 --- /dev/null +++ b/optd-mvp/src/entities/group.rs @@ -0,0 +1,82 @@ +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "group")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i32, + pub status: i8, + pub winner: Option, + pub cost: Option, + pub set_size: i32, + pub parent_id: Option, + pub next_id: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "Entity", + from = "Column::NextId", + to = "Column::Id", + on_update = "Cascade", + on_delete = "SetNull" + )] + SelfRef2, + #[sea_orm( + belongs_to = "Entity", + from = "Column::ParentId", + to = "Column::Id", + on_update = "Cascade", + on_delete = "SetNull" + )] + SelfRef1, + #[sea_orm(has_many = "super::logical_children::Entity")] + LogicalChildren, + #[sea_orm(has_many = "super::logical_expression::Entity")] + LogicalExpression, + #[sea_orm(has_many = "super::physical_children::Entity")] + PhysicalChildren, + #[sea_orm( + belongs_to = "super::physical_expression::Entity", + from = "Column::Winner", + to = "super::physical_expression::Column::Id", + on_update = "Cascade", + on_delete = "SetNull" + )] + PhysicalExpression, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::LogicalChildren.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::PhysicalChildren.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + super::logical_children::Relation::LogicalExpression.def() + } + fn via() -> Option { + Some(super::logical_children::Relation::Group.def().rev()) + } +} + +impl Related for Entity { + fn to() -> RelationDef { + super::physical_children::Relation::PhysicalExpression.def() + } + fn via() -> Option { + Some(super::physical_children::Relation::Group.def().rev()) + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/optd-mvp/src/entities/logical_children.rs b/optd-mvp/src/entities/logical_children.rs new file mode 100644 index 0000000..31e86c9 --- /dev/null +++ b/optd-mvp/src/entities/logical_children.rs @@ -0,0 +1,46 @@ +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "logical_children")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub logical_expression_id: i32, + #[sea_orm(primary_key, auto_increment = false)] + pub group_id: i32, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::group::Entity", + from = "Column::GroupId", + to = "super::group::Column::Id", + on_update = "Cascade", + on_delete = "Cascade" + )] + Group, + #[sea_orm( + belongs_to = "super::logical_expression::Entity", + from = "Column::LogicalExpressionId", + to = "super::logical_expression::Column::Id", + on_update = "Cascade", + on_delete = "Cascade" + )] + LogicalExpression, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Group.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::LogicalExpression.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/optd-mvp/src/entities/logical_expression.rs b/optd-mvp/src/entities/logical_expression.rs new file mode 100644 index 0000000..9311776 --- /dev/null +++ b/optd-mvp/src/entities/logical_expression.rs @@ -0,0 +1,56 @@ +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "logical_expression")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i32, + pub group_id: i32, + pub kind: i16, + pub data: Json, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm(has_many = "super::fingerprint::Entity")] + Fingerprint, + #[sea_orm( + belongs_to = "super::group::Entity", + from = "Column::GroupId", + to = "super::group::Column::Id", + on_update = "Cascade", + on_delete = "Cascade" + )] + Group, + #[sea_orm(has_many = "super::logical_children::Entity")] + LogicalChildren, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Fingerprint.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::LogicalChildren.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + super::logical_children::Relation::Group.def() + } + fn via() -> Option { + Some( + super::logical_children::Relation::LogicalExpression + .def() + .rev(), + ) + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/optd-mvp/src/entities/mod.rs b/optd-mvp/src/entities/mod.rs new file mode 100644 index 0000000..8bdd8f7 --- /dev/null +++ b/optd-mvp/src/entities/mod.rs @@ -0,0 +1,10 @@ +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.2 + +pub mod prelude; + +pub mod fingerprint; +pub mod group; +pub mod logical_children; +pub mod logical_expression; +pub mod physical_children; +pub mod physical_expression; diff --git a/optd-mvp/src/entities/physical_children.rs b/optd-mvp/src/entities/physical_children.rs new file mode 100644 index 0000000..94859a6 --- /dev/null +++ b/optd-mvp/src/entities/physical_children.rs @@ -0,0 +1,46 @@ +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "physical_children")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub physical_expression_id: i32, + #[sea_orm(primary_key, auto_increment = false)] + pub group_id: i32, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::group::Entity", + from = "Column::GroupId", + to = "super::group::Column::Id", + on_update = "Cascade", + on_delete = "Cascade" + )] + Group, + #[sea_orm( + belongs_to = "super::physical_expression::Entity", + from = "Column::PhysicalExpressionId", + to = "super::physical_expression::Column::Id", + on_update = "Cascade", + on_delete = "Cascade" + )] + PhysicalExpression, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Group.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::PhysicalExpression.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/optd-mvp/src/entities/physical_expression.rs b/optd-mvp/src/entities/physical_expression.rs new file mode 100644 index 0000000..918ac03 --- /dev/null +++ b/optd-mvp/src/entities/physical_expression.rs @@ -0,0 +1,48 @@ +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "physical_expression")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i32, + pub group_id: i32, + pub kind: i16, + pub data: Json, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::group::Entity", + from = "Column::GroupId", + to = "super::group::Column::Id", + on_update = "Cascade", + on_delete = "Cascade" + )] + Group, + #[sea_orm(has_many = "super::physical_children::Entity")] + PhysicalChildren, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::PhysicalChildren.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + super::physical_children::Relation::Group.def() + } + fn via() -> Option { + Some( + super::physical_children::Relation::PhysicalExpression + .def() + .rev(), + ) + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/optd-mvp/src/entities/prelude.rs b/optd-mvp/src/entities/prelude.rs new file mode 100644 index 0000000..1e27a2c --- /dev/null +++ b/optd-mvp/src/entities/prelude.rs @@ -0,0 +1,10 @@ +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.2 + +#![allow(unused_imports)] + +pub use super::fingerprint::Entity as Fingerprint; +pub use super::group::Entity as Group; +pub use super::logical_children::Entity as LogicalChildren; +pub use super::logical_expression::Entity as LogicalExpression; +pub use super::physical_children::Entity as PhysicalChildren; +pub use super::physical_expression::Entity as PhysicalExpression; diff --git a/optd-mvp/src/expression/logical_expression.rs b/optd-mvp/src/expression/logical_expression.rs new file mode 100644 index 0000000..486f1cb --- /dev/null +++ b/optd-mvp/src/expression/logical_expression.rs @@ -0,0 +1,223 @@ +//! Definition of logical expressions / relations in our query optimization framework. +//! +//! FIXME: All fields are placeholders. +//! +//! TODO Figure out if each relation should be in a different submodule. +//! TODO This entire file is a WIP. + +use crate::{entities::logical_expression::Model, memo::GroupId}; +use fxhash::hash; +use serde::{Deserialize, Serialize}; +use std::fmt::Debug; + +/// An interface defining what an in-memory logical expression representation should be able to do. +pub trait LogicalExpression: From + Into + Clone + Debug { + /// Returns the kind of relation / operator node encoded as an integer. + fn kind(&self) -> i16; + + /// Retrieves the child groups IDs of this logical expression. + fn children(&self) -> Vec; + + /// Computes the fingerprint of this expression, which should generate an integer for equality + /// checks that has a low collision rate. + fn fingerprint(&self) -> i64; + + /// Checks if the current expression is a duplicate of the other expression. + /// + /// Note that this is similar to `Eq` and `PartialEq`, but the implementor should be aware that + /// different expressions can be duplicates of each other without having the exact same data. + fn is_duplicate(&self, other: &Self) -> bool; + + /// Rewrites the expression to use new child groups IDs, where `rewrites` is a slice of tuples + /// representing `(old_group_id, new_group_id)`. + /// + /// TODO: There's definitely a better way to represent this API + fn rewrite(&self, rewrites: &[(GroupId, GroupId)]) -> Self; +} + +#[derive(Clone, Debug)] +pub enum DefaultLogicalExpression { + Scan(Scan), + Filter(Filter), + Join(Join), +} + +impl LogicalExpression for DefaultLogicalExpression { + fn kind(&self) -> i16 { + match self { + Self::Scan(_) => 0, + Self::Filter(_) => 1, + Self::Join(_) => 2, + } + } + + fn children(&self) -> Vec { + match self { + Self::Scan(_) => vec![], + Self::Filter(filter) => vec![filter.child], + Self::Join(join) => vec![join.left, join.right], + } + } + + fn fingerprint(&self) -> i64 { + let kind = self.kind() as u16 as usize; + let hash = match self { + Self::Scan(scan) => hash(scan.table.as_str()), + Self::Filter(filter) => hash(&filter.child.0) ^ hash(filter.expression.as_str()), + Self::Join(join) => { + // Make sure that there is a difference between `Join(A, B)` and `Join(B, A)`. + hash(&(join.left.0 + 1)) + ^ hash(&(join.right.0 + 2)) + ^ hash(join.expression.as_str()) + } + }; + + // Mask out the bottom 16 bits of `hash` and replace them with `kind`. + ((hash & !0xFFFF) | kind) as i64 + } + + fn is_duplicate(&self, other: &Self) -> bool { + match (self, other) { + (Self::Scan(scan_left), Self::Scan(scan_right)) => scan_left.table == scan_right.table, + (Self::Filter(filter_left), Self::Filter(filter_right)) => { + filter_left.child == filter_right.child + && filter_left.expression == filter_right.expression + } + (Self::Join(join_left), Self::Join(join_right)) => { + join_left.left == join_right.left + && join_left.right == join_right.right + && join_left.expression == join_right.expression + } + _ => false, + } + } + + fn rewrite(&self, rewrites: &[(GroupId, GroupId)]) -> Self { + let rewrite = |x: GroupId| { + if rewrites.is_empty() { + return x; + } + + if let Some(i) = rewrites.iter().position(|(curr, _new)| &x == curr) { + assert_eq!(rewrites[i].0, x); + rewrites[i].1 + } else { + x + } + }; + + match self { + Self::Scan(_) => self.clone(), + Self::Filter(filter) => Self::Filter(Filter { + child: rewrite(filter.child), + expression: filter.expression.clone(), + }), + Self::Join(join) => Self::Join(Join { + left: rewrite(join.left), + right: rewrite(join.right), + expression: join.expression.clone(), + }), + } + } +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct Scan { + table: String, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct Filter { + child: GroupId, + expression: String, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct Join { + left: GroupId, + right: GroupId, + expression: String, +} + +impl From for DefaultLogicalExpression { + fn from(value: Model) -> Self { + match value.kind { + 0 => Self::Scan( + serde_json::from_value(value.data) + .expect("unable to deserialize data into a logical `Scan`"), + ), + 1 => Self::Filter( + serde_json::from_value(value.data) + .expect("Unable to deserialize data into a logical `Filter`"), + ), + 2 => Self::Join( + serde_json::from_value(value.data) + .expect("Unable to deserialize data into a logical `Join`"), + ), + _ => panic!(), + } + } +} + +impl From for Model { + fn from(value: DefaultLogicalExpression) -> Model { + fn create_logical_expression(kind: i16, data: serde_json::Value) -> Model { + Model { + id: -1, + group_id: -1, + kind, + data, + } + } + + let kind = value.kind(); + match value { + DefaultLogicalExpression::Scan(scan) => create_logical_expression( + kind, + serde_json::to_value(scan).expect("unable to serialize logical `Scan`"), + ), + DefaultLogicalExpression::Filter(filter) => create_logical_expression( + kind, + serde_json::to_value(filter).expect("unable to serialize logical `Filter`"), + ), + DefaultLogicalExpression::Join(join) => create_logical_expression( + kind, + serde_json::to_value(join).expect("unable to serialize logical `Join`"), + ), + } + } +} + +#[cfg(test)] +pub use build::*; + +#[cfg(test)] +mod build { + use super::*; + use crate::expression::DefaultLogicalExpression; + + pub fn scan(table_schema: String) -> DefaultLogicalExpression { + DefaultLogicalExpression::Scan(Scan { + table: table_schema, + }) + } + + pub fn filter(child_group: GroupId, expression: String) -> DefaultLogicalExpression { + DefaultLogicalExpression::Filter(Filter { + child: child_group, + expression, + }) + } + + pub fn join( + left_group: GroupId, + right_group: GroupId, + expression: String, + ) -> DefaultLogicalExpression { + DefaultLogicalExpression::Join(Join { + left: left_group, + right: right_group, + expression, + }) + } +} diff --git a/optd-mvp/src/expression/mod.rs b/optd-mvp/src/expression/mod.rs new file mode 100644 index 0000000..0efbb44 --- /dev/null +++ b/optd-mvp/src/expression/mod.rs @@ -0,0 +1,9 @@ +//! In-memory representation of logical and physical expression / operators / relations. +//! +//! TODO more docs. + +mod logical_expression; +pub use logical_expression::*; + +mod physical_expression; +pub use physical_expression::*; diff --git a/optd-mvp/src/expression/physical_expression.rs b/optd-mvp/src/expression/physical_expression.rs new file mode 100644 index 0000000..d7f71de --- /dev/null +++ b/optd-mvp/src/expression/physical_expression.rs @@ -0,0 +1,126 @@ +//! Definition of physical expressions / operators in our query optimization framework. +//! +//! FIXME: All fields are placeholders. +//! +//! TODO Remove dead code. +//! TODO Figure out if each operator should be in a different submodule. +//! TODO This entire file is a WIP. + +#![allow(dead_code)] + +use crate::{entities::physical_expression::Model, memo::GroupId}; +use serde::{Deserialize, Serialize}; +use std::fmt::Debug; + +/// An interface defining what an in-memory physical expression representation should be able to do. +pub trait PhysicalExpression: From + Into + Clone + Debug { + /// Returns the kind of relation / operator node encoded as an integer. + fn kind(&self) -> i16; + + /// Retrieves the child groups IDs of this logical expression. + fn children(&self) -> Vec; +} + +impl PhysicalExpression for DefaultPhysicalExpression { + fn kind(&self) -> i16 { + match self { + Self::TableScan(_) => 0, + Self::Filter(_) => 1, + Self::HashJoin(_) => 2, + } + } + + fn children(&self) -> Vec { + match self { + Self::TableScan(_) => vec![], + Self::Filter(filter) => vec![filter.child], + Self::HashJoin(hash_join) => vec![hash_join.left, hash_join.right], + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DefaultPhysicalExpression { + TableScan(TableScan), + Filter(PhysicalFilter), + HashJoin(HashJoin), +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] +pub struct TableScan { + table_schema: String, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] +pub struct PhysicalFilter { + child: GroupId, + expression: String, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] +pub struct HashJoin { + left: GroupId, + right: GroupId, + expression: String, +} + +impl From for DefaultPhysicalExpression { + fn from(value: Model) -> Self { + match value.kind { + 0 => Self::TableScan( + serde_json::from_value(value.data) + .expect("unable to deserialize data into a physical `TableScan`"), + ), + 1 => Self::Filter( + serde_json::from_value(value.data) + .expect("Unable to deserialize data into a physical `Filter`"), + ), + 2 => Self::HashJoin( + serde_json::from_value(value.data) + .expect("Unable to deserialize data into a physical `HashJoin`"), + ), + _ => panic!(), + } + } +} + +impl From for Model { + fn from(value: DefaultPhysicalExpression) -> Model { + fn create_physical_expression(kind: i16, data: serde_json::Value) -> Model { + Model { + id: -1, + group_id: -1, + kind, + data, + } + } + + match value { + DefaultPhysicalExpression::TableScan(scan) => create_physical_expression( + 0, + serde_json::to_value(scan).expect("unable to serialize physical `TableScan`"), + ), + DefaultPhysicalExpression::Filter(filter) => create_physical_expression( + 1, + serde_json::to_value(filter).expect("unable to serialize physical `Filter`"), + ), + DefaultPhysicalExpression::HashJoin(join) => create_physical_expression( + 2, + serde_json::to_value(join).expect("unable to serialize physical `HashJoin`"), + ), + } + } +} + +#[cfg(test)] +pub use build::*; + +#[cfg(test)] +mod build { + use super::*; + use crate::expression::DefaultPhysicalExpression; + + pub fn table_scan(table_schema: String) -> DefaultPhysicalExpression { + DefaultPhysicalExpression::TableScan(TableScan { table_schema }) + } +} diff --git a/optd-mvp/src/lib.rs b/optd-mvp/src/lib.rs new file mode 100644 index 0000000..2e238e0 --- /dev/null +++ b/optd-mvp/src/lib.rs @@ -0,0 +1,45 @@ +//! This crate is an attempt to make an MVP of a duplicate-detecting memo table for query +//! optimization. +//! +//! TODO write more docs. + +use sea_orm::*; +use sea_orm_migration::prelude::*; +use thiserror::Error; + +mod migrator; +use migrator::Migrator; + +mod entities; + +pub mod expression; +pub mod memo; + +/// The filename of the SQLite database for migration. +pub const DATABASE_FILENAME: &str = "sqlite.db"; +/// The URL of the SQLite database for migration. +pub const DATABASE_URL: &str = "sqlite:./sqlite.db?mode=rwc"; + +/// An error type wrapping all the different kinds of error the optimizer might raise. +#[allow(missing_docs)] +#[derive(Error, Debug)] +pub enum OptimizerError { + #[error("SeaORM error")] + Database(#[from] sea_orm::error::DbErr), + #[error("Memo table logical error")] + Memo(#[from] memo::MemoError), + #[error("unknown error")] + Unknown, +} + +/// Shorthand for a [`Result`] with an error type [`OptimizerError`]. +pub type OptimizerResult = Result; + +/// Applies all migrations. +/// +/// # Errors +/// +/// Returns a [`DbErr`] if unable to apply any migrations. +pub async fn migrate(db: &DatabaseConnection) -> Result<(), DbErr> { + Migrator::refresh(db).await +} diff --git a/optd-mvp/src/main.rs b/optd-mvp/src/main.rs new file mode 100644 index 0000000..e7a11a9 --- /dev/null +++ b/optd-mvp/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/optd-mvp/src/memo/mod.rs b/optd-mvp/src/memo/mod.rs new file mode 100644 index 0000000..690bdee --- /dev/null +++ b/optd-mvp/src/memo/mod.rs @@ -0,0 +1,52 @@ +//! This module contains items related to the memo table. +//! +//! TODO more docs. + +#![warn(missing_docs)] +#![warn(clippy::missing_docs_in_private_items)] +#![warn(clippy::missing_errors_doc)] +#![warn(clippy::missing_panics_doc)] +#![warn(clippy::missing_safety_doc)] + +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +/// A new type of an integer identifying a unique group. +#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)] +#[serde(transparent)] +pub struct GroupId(pub i32); + +/// A new type of an integer identifying a unique logical expression. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LogicalExpressionId(pub i32); + +/// A new type of an integer identifying a unique physical expression. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PhysicalExpressionId(pub i32); + +/// A status enum representing the different states a group can be during query optimization. +#[repr(u8)] +pub enum GroupStatus { + /// Represents a group that is currently being logically explored. + InProgress = 0, + /// Represents a logically explored group that is currently being physically optimized. + Explored = 1, + /// Represents a fully optimized group. + Optimized = 2, +} + +/// The different kinds of errors that might occur while running operations on a memo table. +#[allow(missing_docs)] +#[derive(Error, Debug)] +pub enum MemoError { + #[error("unknown group ID {0:?}")] + UnknownGroup(GroupId), + #[error("unknown logical expression ID {0:?}")] + UnknownLogicalExpression(LogicalExpressionId), + #[error("unknown physical expression ID {0:?}")] + UnknownPhysicalExpression(PhysicalExpressionId), + #[error("invalid expression encountered")] + InvalidExpression, +} + +pub mod persistent; diff --git a/optd-mvp/src/memo/persistent/implementation.rs b/optd-mvp/src/memo/persistent/implementation.rs new file mode 100644 index 0000000..9ce30ce --- /dev/null +++ b/optd-mvp/src/memo/persistent/implementation.rs @@ -0,0 +1,764 @@ +//! This module contains the implementation of [`PersistentMemo`]. +//! +//! TODO For parallelism, almost all of these methods need to be under transactions. +//! TODO Write more docs. +//! TODO Remove dead code. + +#![allow(dead_code)] + +use super::{PersistentMemo, PersistentMemoTransaction}; +use crate::{ + entities::*, + expression::{LogicalExpression, PhysicalExpression}, + memo::{GroupId, GroupStatus, LogicalExpressionId, MemoError, PhysicalExpressionId}, + OptimizerResult, DATABASE_URL, +}; +use sea_orm::{ + entity::{prelude::*, IntoActiveModel, NotSet, Set}, + Database, DatabaseTransaction, TransactionTrait, +}; +use std::{collections::HashSet, marker::PhantomData}; + +impl PersistentMemo +where + L: LogicalExpression, + P: PhysicalExpression, +{ + /// Creates a new `PersistentMemo` struct by connecting to a database defined at + /// [`DATABASE_URL`]. + /// + /// # Panics + /// + /// Panics if unable to create a databse connection to [`DATABASE_URL`]. + pub async fn new() -> Self { + Self { + db: Database::connect(DATABASE_URL).await.unwrap(), + _phantom_logical: PhantomData, + _phantom_physical: PhantomData, + } + } + + /// Starts a new database transaction. + /// + /// # Errors + /// + /// Returns a [`DbErr`] if unable to create a new transaction. + pub async fn begin(&self) -> OptimizerResult> { + Ok(PersistentMemoTransaction::new(self.db.begin().await?).await) + } + + /// Deletes all objects in the backing database. + /// + /// Since there is no asynchronous drop yet in Rust, in order to drop all objects in the + /// database, the user must call this manually. + /// + /// # Panics + /// + /// May panic if unable to delete entities from any table. + pub async fn cleanup(&self) { + /// Simple private macro to teardown all tables in the database. + /// Note that these have to be specified manually, so when adding a new table to the + /// database, we must make sure to add that table here. + macro_rules! delete_all { + ($($module: ident),+ $(,)?) => { + $( + $module::Entity::delete_many() + .exec(&self.db) + .await + .unwrap(); + )+ + }; + } + + delete_all! { + group, + fingerprint, + logical_expression, + logical_children, + physical_expression, + physical_children + }; + } +} + +impl PersistentMemoTransaction +where + L: LogicalExpression, + P: PhysicalExpression, +{ + /// Creates a new transaction object. + pub async fn new(txn: DatabaseTransaction) -> Self { + Self { + txn, + _phantom_logical: PhantomData, + _phantom_physical: PhantomData, + } + } + + /// Commits the transaction. + /// + /// # Errors + /// + /// Returns a [`DbErr`] if unable to commit the transaction. + pub async fn commit(self) -> OptimizerResult<()> { + Ok(self.txn.commit().await?) + } + + /// Rolls back the transaction. + /// + /// # Errors + /// + /// Returns a [`DbErr`] if unable to roll back the transaction. + pub async fn rollback(self) -> OptimizerResult<()> { + Ok(self.txn.rollback().await?) + } + + /// Retrieves a [`group::Model`] given its ID. + /// + /// FIXME: use an in-memory representation of a group instead. + /// + /// # Errors + /// + /// If the group does not exist, returns a [`MemoError::UnknownGroup`] error. + pub async fn get_group(&self, group_id: GroupId) -> OptimizerResult { + Ok(group::Entity::find_by_id(group_id.0) + .one(&self.txn) + .await? + .ok_or(MemoError::UnknownGroup(group_id))?) + } + + /// Retrieves the root / canonical group ID of the given group ID. + /// + /// The groups form a union find / disjoint set parent pointer forest, where group merging + /// causes two trees to merge. + /// + /// This function uses the path compression optimization, which amortizes the cost to a single + /// lookup (theoretically in constant time, but we must be wary of the I/O roundtrip). + /// + /// # Errors + /// + /// If the group does not exist, returns a [`MemoError::UnknownGroup`] error. This function + /// also performs path compression pointer updates, so any of those updates can fail with a + /// [`DbErr`]. + pub async fn get_root_group(&self, group_id: GroupId) -> OptimizerResult { + let curr_group = self.get_group(group_id).await?; + + // If we have no parent, then we are at the root. + let Some(parent_id) = curr_group.parent_id else { + return Ok(GroupId(curr_group.id)); + }; + + // Recursively find the root group ID. + let root_id = Box::pin(self.get_root_group(GroupId(parent_id))).await?; + + // Path Compression Optimization: + // For every group along the path that we walked, set their parent id pointer to the root. + // This allows for an amortized O(1) cost for `get_root_group`. + let mut active_group = curr_group.into_active_model(); + + // Update the group to point to the new parent. + active_group.parent_id = Set(Some(root_id.0)); + active_group.update(&self.txn).await?; + + Ok(GroupId(root_id.0)) + } + + /// Retrieves every group ID of groups that share the same root group with the input group. + /// + /// The group records form a union-find data structure that also maintains a circular linked + /// list in every set that allows us to iterate over all elements in a set in linear time. + /// + /// # Errors + /// + /// If the input group does not exist, or if any pointer along the path is invalid, returns a + /// [`MemoError::UnknownGroup`] error. + /// + /// # Panics + /// + /// Panics if the embedded union-find data structure is malformed. + pub async fn get_group_set(&self, group_id: GroupId) -> OptimizerResult> { + // Iterate over the circular linked list until we reach ourselves again. + let base_group = self.get_group(group_id).await?; + + // The only case when `next_id` is set to `None` is if the current group is a root, which + // means that this group is the only group in the set. + if base_group.next_id.is_none() { + assert!(base_group.parent_id.is_none()); + return Ok(vec![group_id]); + } + + // Iterate over the circular linked list until we see ourselves again, collecting nodes + // along the way. + let mut set = vec![group_id]; + let mut next_id = base_group + .next_id + .expect("next pointer cannot be null if it is in a cycle"); + loop { + let curr_group = self.get_group(GroupId(next_id)).await?; + + if curr_group.id == group_id.0 { + break; + } + + set.push(GroupId(curr_group.id)); + next_id = curr_group + .next_id + .expect("next pointer cannot be null if it is in a cycle"); + } + + Ok(set) + } + + /// Retrieves a [`physical_expression::Model`] given a [`PhysicalExpressionId`]. + /// + /// # Errors + /// + /// If the physical expression does not exist, returns a + /// [`MemoError::UnknownPhysicalExpression`] error. + pub async fn get_physical_expression( + &self, + physical_expression_id: PhysicalExpressionId, + ) -> OptimizerResult<(GroupId, P)> { + // Lookup the entity in the database via the unique expression ID. + let model = physical_expression::Entity::find_by_id(physical_expression_id.0) + .one(&self.txn) + .await? + .ok_or(MemoError::UnknownPhysicalExpression(physical_expression_id))?; + + let group_id = GroupId(model.group_id); + let expr = model.into(); + + Ok((group_id, expr)) + } + + /// Retrieves a [`logical_expression::Model`] given its [`LogicalExpressionId`]. + /// + /// # Errors + /// + /// If the logical expression does not exist, returns a [`MemoError::UnknownLogicalExpression`] + /// error. + pub async fn get_logical_expression( + &self, + logical_expression_id: LogicalExpressionId, + ) -> OptimizerResult<(GroupId, L)> { + // Lookup the entity in the database via the unique expression ID. + let model = logical_expression::Entity::find_by_id(logical_expression_id.0) + .one(&self.txn) + .await? + .ok_or(MemoError::UnknownLogicalExpression(logical_expression_id))?; + + let group_id = GroupId(model.group_id); + let expr = model.into(); + + Ok((group_id, expr)) + } + + /// Retrieves all of the logical expression "children" IDs of a group. + /// + /// FIXME: `find_related` does not work for some reason, have to use manual `filter`. + /// + /// # Errors + /// + /// If the group does not exist, returns a [`MemoError::UnknownGroup`] error. Can also return + /// a [`DbErr`] if the something goes wrong with the filter scan. + pub async fn get_logical_children( + &self, + group_id: GroupId, + ) -> OptimizerResult> { + // First ensure that the group exists. + let _ = self.get_group(group_id).await?; + + // Search for expressions that have the given parent group ID. + let children = logical_expression::Entity::find() + .filter(logical_expression::Column::GroupId.eq(group_id.0)) + .all(&self.txn) + .await? + .into_iter() + .map(|m| LogicalExpressionId(m.id)) + .collect(); + + Ok(children) + } + + /// Retrieves all of the physical expression "children" IDs of a group. + /// + /// FIXME: `find_related` does not work for some reason, have to use manual `filter`. + /// + /// # Errors + /// + /// If the group does not exist, returns a [`MemoError::UnknownGroup`] error. Can also return + /// a [`DbErr`] if the something goes wrong with the filter scan. + pub async fn get_physical_children( + &self, + group_id: GroupId, + ) -> OptimizerResult> { + // First ensure that the group exists. + let _ = self.get_group(group_id).await?; + + // Search for expressions that have the given parent group ID. + let children = physical_expression::Entity::find() + .filter(physical_expression::Column::GroupId.eq(group_id.0)) + .all(&self.txn) + .await? + .into_iter() + .map(|m| PhysicalExpressionId(m.id)) + .collect(); + + Ok(children) + } + + /// Updates / replaces a group's status. Returns the previous group status. + /// + /// # Errors + /// + /// If the group does not exist, returns a [`MemoError::UnknownGroup`] error. Can also return a + /// [`DbErr`] if the update fails. + pub async fn update_group_status( + &mut self, + group_id: GroupId, + status: GroupStatus, + ) -> OptimizerResult { + // First retrieve the group record. + let mut group = self.get_group(group_id).await?.into_active_model(); + + // Update the group's status. + let old_status = group.status; + group.status = Set(status as u8 as i8); + group.update(&self.txn).await?; + + let old_status = match old_status.unwrap() { + 0 => GroupStatus::InProgress, + 1 => GroupStatus::Explored, + 2 => GroupStatus::Optimized, + _ => unreachable!("encountered an invalid group status"), + }; + + Ok(old_status) + } + + /// Updates / replaces a group's best physical plan (winner). Optionally returns the previous + /// winner's physical expression ID. + /// + /// FIXME: In the future, this should first check that we aren't overwriting a winner that was + /// updated from another thread by comparing against the cost of the plan. + /// + /// # Errors + /// + /// If the group does not exist, returns a [`MemoError::UnknownGroup`] error. Can also return a + /// [`DbErr`] if the update fails. + pub async fn update_group_winner( + &mut self, + group_id: GroupId, + physical_expression_id: PhysicalExpressionId, + ) -> OptimizerResult> { + // First retrieve the group record. + let mut group = self.get_group(group_id).await?.into_active_model(); + + // Update the group to point to the new winner. + let old_id = group.winner; + group.winner = Set(Some(physical_expression_id.0)); + group.update(&self.txn).await?; + + // Note that the `unwrap` here is unwrapping the `ActiveValue`, not the `Option`. + let old_id = old_id.unwrap().map(PhysicalExpressionId); + Ok(old_id) + } + + /// Adds a logical expression to an existing group via its ID. + /// + /// The caller is required to pass in a slice of [`GroupId`] that represent the child groups of + /// the input expression. + /// + /// If the group does not exist, returns a [`MemoError::UnknownGroup`] error. + /// + /// If the memo table detects that the input logical expression is a duplicate expression, this + /// function will **not** insert the expression into the memo table. Instead, it will return an + /// `Ok(Err(expression_id))`, which is a unique identifier of the expression that the input is a + /// duplicate of. The caller can use this ID to retrieve the group the original belongs to. + /// + /// If the memo table detects that the input is unique, it will insert the expression into the + /// input group and return an `Ok(Ok(expression_id))`. + /// + /// # Errors + /// + /// Note that the return value is a [`Result`] wrapped in an [`OptimizerResult`]. The outer + /// result is used for raising [`DbErr`] or other database/IO-related errors. The inner result + /// is used for notifying the caller if the expression that they attempted to insert was a + /// duplicate expression or not. + pub async fn add_logical_expression_to_group( + &mut self, + group_id: GroupId, + logical_expression: L, + children: &[GroupId], + ) -> OptimizerResult> { + // Check if the expression already exists anywhere in the memo table. + if let Some(existing_id) = self + .is_duplicate_logical_expression(&logical_expression, children) + .await? + { + return Ok(Err(existing_id)); + } + + // Check if the group actually exists. + let _ = self.get_group(group_id).await?; + + // Insert the expression. + let model: logical_expression::Model = logical_expression.into(); + let mut active_model = model.into_active_model(); + active_model.group_id = Set(group_id.0); + active_model.id = NotSet; + let new_model = active_model.insert(&self.txn).await?; + + let expr_id = new_model.id; + + // Insert the child groups of the expression into the junction / children table. + logical_children::Entity::insert_many(children.iter().copied().map(|child_id| { + logical_children::ActiveModel { + logical_expression_id: Set(expr_id), + group_id: Set(child_id.0), + } + })) + .on_empty_do_nothing() + .exec(&self.txn) + .await?; + + // Finally, insert the fingerprint of the logical expression as well. + let new_expr: L = new_model.into(); + let kind = new_expr.kind(); + + // In order to calculate a correct fingerprint, we will want to use the IDs of the root + // groups of the children instead of the child ID themselves. + let mut rewrites = vec![]; + for &child_id in children { + let root_id = self.get_root_group(child_id).await?; + rewrites.push((child_id, root_id)); + } + let hash = new_expr.rewrite(&rewrites).fingerprint(); + + let fingerprint = fingerprint::ActiveModel { + id: NotSet, + logical_expression_id: Set(expr_id), + kind: Set(kind), + hash: Set(hash), + }; + fingerprint::Entity::insert(fingerprint) + .exec(&self.txn) + .await?; + + Ok(Ok(LogicalExpressionId(expr_id))) + } + + /// Adds a physical expression to an existing group via its ID. + /// + /// The caller is required to pass in a slice of [`GroupId`] that represent the child groups of + /// the input expression. + /// + /// On successful insertion, returns the ID of the physical expression. + /// + /// # Errors + /// + /// If the group does not exist, returns a [`MemoError::UnknownGroup`] error. Can also fail if + /// insertion of the new physical expression or any of its child junction entries are not able + /// to be inserted. + pub async fn add_physical_expression_to_group( + &mut self, + group_id: GroupId, + physical_expression: P, + children: &[GroupId], + ) -> OptimizerResult { + // Check if the group actually exists. + let _ = self.get_group(group_id).await?; + + // Insert the expression. + let model: physical_expression::Model = physical_expression.into(); + let mut active_model = model.into_active_model(); + active_model.group_id = Set(group_id.0); + active_model.id = NotSet; + let new_model = active_model.insert(&self.txn).await?; + + // Insert the child groups of the expression into the junction / children table. + physical_children::Entity::insert_many(children.iter().copied().map(|child_id| { + physical_children::ActiveModel { + physical_expression_id: Set(new_model.id), + group_id: Set(child_id.0), + } + })) + .on_empty_do_nothing() + .exec(&self.txn) + .await?; + + Ok(PhysicalExpressionId(new_model.id)) + } + + /// Checks if the given logical expression is a duplicate / already exists in the memo table. + /// + /// In order to prevent a large amount of duplicate work, the memo table must support duplicate + /// expression detection. + /// + /// Returns `Some((group_id, expression_id))` if the memo table detects that the expression + /// already exists, and `None` otherwise. + /// + /// This function assumes that the child groups of the expression are currently roots of their + /// group sets. For example, if G1 and G2 should be merged, and G1 is the root, then the input + /// expression should _not_ have G2 as a child, and should be replaced with G1. + /// + /// # Errors + /// + /// Returns a [`DbErr`] when a database operation fails. + pub async fn is_duplicate_logical_expression( + &self, + logical_expression: &L, + children: &[GroupId], + ) -> OptimizerResult> { + let model: logical_expression::Model = logical_expression.clone().into(); + + // Lookup all expressions that have the same fingerprint and kind. There may be false + // positives, but we will check for those next. + let kind = model.kind; + + // In order to calculate a correct fingerprint, we will want to use the IDs of the root + // groups of the children instead of the child ID themselves. + let mut rewrites = vec![]; + for &child_id in children { + let root_id = self.get_root_group(child_id).await?; + rewrites.push((child_id, root_id)); + } + let fingerprint = logical_expression.rewrite(&rewrites).fingerprint(); + + // Filter first by the fingerprint, and then the kind. + // FIXME: The kind is already embedded into the fingerprint, so we may not actually need the + // second filter? + let potential_matches = fingerprint::Entity::find() + .filter(fingerprint::Column::Hash.eq(fingerprint)) + .filter(fingerprint::Column::Kind.eq(kind)) + .all(&self.txn) + .await?; + + if potential_matches.is_empty() { + return Ok(None); + } + + // Now that we have all of the expressions that match the given fingerprint, we need to + // filter out all of the expressions that might have had the same fingerprint but are not + // actually equivalent (hash collisions). + let mut match_id = None; + for potential_match in potential_matches { + let expr_id = LogicalExpressionId(potential_match.logical_expression_id); + let (group_id, expr) = self.get_logical_expression(expr_id).await?; + + // We need to add the root groups of the new expression to the rewrites vector. + // TODO make this much more efficient by making rewrites a hash map, potentially im::HashMap. + let mut rewrites = rewrites.clone(); + for child_id in expr.children() { + let root_id = self.get_root_group(child_id).await?; + rewrites.push((child_id, root_id)); + } + + // Check for an exact match after rewrites. + if logical_expression + .rewrite(&rewrites) + .is_duplicate(&expr.rewrite(&rewrites)) + { + match_id = Some((group_id, expr_id)); + + // There should be at most one duplicate expression, so we can break here. + break; + } + } + + Ok(match_id) + } + + /// Adds a new group into the memo table via a logical expression, creating a new group if the + /// logical expression does not already exist. + /// + /// The caller is required to pass in a slice of [`GroupId`] that represent the child groups of + /// the input expression. + /// + /// If the expression already exists, then this function will return the [`GroupId`] of the + /// parent group and the corresponding (already existing) [`LogicalExpressionId`]. It will also + /// completely ignore the group ID field of the input expression as well as ignore the input + /// slice of child groups. + /// + /// If the expression does not exist, this function will create a new group and a new + /// expression, returning brand new IDs for both. + /// + /// # Errors + /// + /// Note that the return value is a [`Result`] wrapped in an [`OptimizerResult`]. The outer + /// result is used for raising [`DbErr`] or other database/IO-related errors. The inner result + /// is used for notifying the caller if the expression/group that they attempted to insert was a + /// duplicate expression or not. + pub async fn add_group( + &mut self, + logical_expression: L, + children: &[GroupId], + ) -> OptimizerResult> + { + // Check if the expression already exists in the memo table. + if let Some((group_id, existing_id)) = self + .is_duplicate_logical_expression(&logical_expression, children) + .await? + { + return Ok(Err((group_id, existing_id))); + } + + // The expression does not exist yet, so we need to create a new group and new expression. + let group = group::ActiveModel { + status: Set(0), // `GroupStatus::InProgress` status. + set_size: Set(1), + ..Default::default() + }; + + // Create the new group. + let group_res = group::Entity::insert(group).exec(&self.txn).await?; + let group_id = group_res.last_insert_id; + + // Insert the input expression into the newly created group. + let expression: logical_expression::Model = logical_expression.clone().into(); + let mut active_expression = expression.into_active_model(); + active_expression.group_id = Set(group_id); + active_expression.id = NotSet; + let new_expression = active_expression.insert(&self.txn).await?; + + let group_id = new_expression.group_id; + let expr_id = new_expression.id; + + // Insert the child groups of the expression into the junction / children table. + logical_children::Entity::insert_many(children.iter().copied().map(|child_id| { + logical_children::ActiveModel { + logical_expression_id: Set(new_expression.id), + group_id: Set(child_id.0), + } + })) + .on_empty_do_nothing() + .exec(&self.txn) + .await?; + + // Finally, insert the fingerprint of the logical expression as well. + let new_logical_expression: L = new_expression.into(); + let kind = new_logical_expression.kind(); + + // In order to calculate a correct fingerprint, we will want to use the IDs of the root + // groups of the children instead of the child ID themselves. + let mut rewrites = vec![]; + for &child_id in children { + let root_id = self.get_root_group(child_id).await?; + rewrites.push((child_id, root_id)); + } + let hash = new_logical_expression.rewrite(&rewrites).fingerprint(); + + let fingerprint = fingerprint::ActiveModel { + id: NotSet, + logical_expression_id: Set(expr_id), + kind: Set(kind), + hash: Set(hash), + }; + fingerprint::Entity::insert(fingerprint) + .exec(&self.txn) + .await?; + + Ok(Ok((GroupId(group_id), LogicalExpressionId(expr_id)))) + } + + /// Merges two groups sets together. + /// + /// If either of the input groups do not exist, returns a [`MemoError::UnknownGroup`] error. + /// + /// TODO write docs. + /// TODO highly inefficient, need to understand metrics and performance testing. + /// TODO Optimization: add rank / size into data structure + /// + /// # Errors + /// + /// TODO + pub async fn merge_groups( + &mut self, + left_group_id: GroupId, + right_group_id: GroupId, + ) -> OptimizerResult { + let mut left_root_id = self.get_root_group(left_group_id).await?; + let mut left_root = self.get_group(left_root_id).await?; + let mut left_size = left_root.set_size; + + let mut right_root_id = self.get_root_group(right_group_id).await?; + let mut right_root = self.get_group(right_root_id).await?; + let mut right_size = left_root.set_size; + + // Rank/size optimization: merge the smaller set into the larger set. + if left_size > right_size { + std::mem::swap(&mut left_root_id, &mut right_root_id); + std::mem::swap(&mut left_root, &mut right_root); + std::mem::swap(&mut left_size, &mut right_size); + } + + // A `None` next pointer means it should technically be pointing to itself. + let left_next = left_root.next_id.unwrap_or(left_root_id.0); + let mut active_left_root = left_root.into_active_model(); + // A `None` next pointer means it should technically be pointing to itself. + let right_next = right_root.next_id.unwrap_or(right_root_id.0); + let mut active_right_root = right_root.into_active_model(); + + // Before we actually update the group records, We first need to generate new fingerprints + // for every single expression that has a child group in the left set. + // TODO make this more efficient, this code is doing double work from `get_group_set`. + let group_set_ids = self.get_group_set(left_group_id).await?; + let mut left_group_models = Vec::with_capacity(group_set_ids.len()); + for &group_id in &group_set_ids { + left_group_models.push(self.get_group(group_id).await?); + } + + // Retrieve every single expression that has a child group in the left set. + let left_group_expressions: Vec> = left_group_models + .load_many_to_many( + logical_expression::Entity, + logical_children::Entity, + &self.txn, + ) + .await?; + + // Need to replace every single occurrence of groups in the set with the new root. + let rewrites: Vec<(GroupId, GroupId)> = group_set_ids + .iter() + .map(|&group_id| (group_id, right_root_id)) + .collect(); + + // For each expression, generate a new fingerprint. + let mut seen = HashSet::new(); + for model in left_group_expressions.into_iter().flatten() { + let expr_id = model.id; + + // There may be duplicates in the expressions list. + if seen.contains(&expr_id) { + continue; + } else { + seen.insert(expr_id); + } + + let logical_expression: L = model.into(); + let hash = logical_expression.rewrite(&rewrites).fingerprint(); + + let fingerprint = fingerprint::ActiveModel { + id: NotSet, + logical_expression_id: Set(expr_id), + kind: Set(logical_expression.kind()), + hash: Set(hash), + }; + fingerprint::Entity::insert(fingerprint) + .exec(&self.txn) + .await?; + } + + // Update the left group root to point to the right group root. + active_left_root.parent_id = Set(Some(right_root_id.0)); + + // Swap the next pointers of each root to maintain the circular linked list. + active_left_root.next_id = Set(Some(right_next)); + active_right_root.next_id = Set(Some(left_next)); + + active_left_root.update(&self.txn).await?; + active_right_root.update(&self.txn).await?; + + Ok(right_root_id) + } +} diff --git a/optd-mvp/src/memo/persistent/mod.rs b/optd-mvp/src/memo/persistent/mod.rs new file mode 100644 index 0000000..dfddffa --- /dev/null +++ b/optd-mvp/src/memo/persistent/mod.rs @@ -0,0 +1,37 @@ +//! This module contains the definition and implementation of the [`PersistentMemo`] type, which +//! implements the `Memo` trait and supports memo table operations necessary for query optimization. + +use sea_orm::{DatabaseConnection, DatabaseTransaction}; +use std::marker::PhantomData; + +#[cfg(test)] +mod tests; + +/// A persistent memo table, backed by a database on disk. +/// +/// TODO more docs. +pub struct PersistentMemo { + /// This `PersistentMemo` is reliant on the SeaORM [`DatabaseConnection`] that stores all of the + /// objects needed for query optimization. + db: DatabaseConnection, + + /// Generic marker for a generic logical expression. + _phantom_logical: PhantomData, + + /// Generic marker for a generic physical expression. + _phantom_physical: PhantomData

, +} + +/// TODO docs. +pub struct PersistentMemoTransaction { + /// A database transaction over the [`PersistentMemo`] table. + txn: DatabaseTransaction, + + /// Generic marker for a generic logical expression. + _phantom_logical: PhantomData, + + /// Generic marker for a generic physical expression. + _phantom_physical: PhantomData

, +} + +pub mod implementation; diff --git a/optd-mvp/src/memo/persistent/tests.rs b/optd-mvp/src/memo/persistent/tests.rs new file mode 100644 index 0000000..8fe1aeb --- /dev/null +++ b/optd-mvp/src/memo/persistent/tests.rs @@ -0,0 +1,483 @@ +use crate::{expression::*, memo::persistent::PersistentMemo}; + +/// Tests that exact expression matches are detected and handled by the txn table. +#[ignore] +#[tokio::test] +async fn test_simple_logical_duplicates() { + let memo = PersistentMemo::::new().await; + memo.cleanup().await; + + let mut txn = memo.begin().await.unwrap(); + + let scan = scan("t1".to_string()); + let scan1a = scan.clone(); + let scan1b = scan.clone(); + let scan2a = scan.clone(); + let scan2b = scan.clone(); + + // Insert a new group and its corresponding expression. + let (group_id, logical_expression_id) = txn.add_group(scan, &[]).await.unwrap().ok().unwrap(); + + // Test `add_logical_expression`. + { + // Attempting to create a new group with a duplicate expression should fail every time. + let (group_id_1a, logical_expression_id_1a) = + txn.add_group(scan1a, &[]).await.unwrap().err().unwrap(); + assert_eq!(group_id, group_id_1a); + assert_eq!(logical_expression_id, logical_expression_id_1a); + + // Try again just in case... + let (group_id_1b, logical_expression_id_1b) = + txn.add_group(scan1b, &[]).await.unwrap().err().unwrap(); + assert_eq!(group_id, group_id_1b); + assert_eq!(logical_expression_id, logical_expression_id_1b); + } + + // Test `add_logical_expression_to_group`. + { + // Attempting to add a duplicate expression into the same group should also fail every time. + let (group_id_2a, logical_expression_id_2a) = txn + .add_logical_expression_to_group(group_id, scan2a, &[]) + .await + .unwrap() + .err() + .unwrap(); + assert_eq!(group_id, group_id_2a); + assert_eq!(logical_expression_id, logical_expression_id_2a); + + let (group_id_2b, logical_expression_id_2b) = txn + .add_logical_expression_to_group(group_id, scan2b, &[]) + .await + .unwrap() + .err() + .unwrap(); + assert_eq!(group_id, group_id_2b); + assert_eq!(logical_expression_id, logical_expression_id_2b); + } + + txn.commit().await.unwrap(); + memo.cleanup().await; +} + +/// Tests that physical expression are _not_ subject to duplicate detection and elimination. +/// +/// !!! Important !!! Note that this behavior should not actually be seen during query optimization, +/// since if logical expression have been deduplicated, there should not be any duplicate physical +/// expressions as they are derivative of the deduplicated logical expressions. +#[ignore] +#[tokio::test] +async fn test_simple_add_physical_expression() { + let memo = PersistentMemo::::new().await; + memo.cleanup().await; + + let mut txn = memo.begin().await.unwrap(); + + // Insert a new group and its corresponding expression. + let scan = scan("t1".to_string()); + let (group_id, _) = txn.add_group(scan, &[]).await.unwrap().ok().unwrap(); + + // Insert two identical physical expressions into the _same_ group. + let table_scan_1 = table_scan("t1".to_string()); + let table_scan_2 = table_scan_1.clone(); + + let physical_expression_id_1 = txn + .add_physical_expression_to_group(group_id, table_scan_1, &[]) + .await + .unwrap(); + + let physical_expression_id_2 = txn + .add_physical_expression_to_group(group_id, table_scan_2, &[]) + .await + .unwrap(); + + // Since physical expressions do not need duplicate detection, + assert_ne!(physical_expression_id_1, physical_expression_id_2); + + txn.commit().await.unwrap(); + memo.cleanup().await; +} + +/// Tests if the txn tables able to correctly retrieve a group's expressions. +#[ignore] +#[tokio::test] +async fn test_simple_tree() { + let memo = PersistentMemo::::new().await; + memo.cleanup().await; + + let mut txn = memo.begin().await.unwrap(); + + // Create two scan groups. + let scan1: DefaultLogicalExpression = scan("t1".to_string()); + let scan2 = scan("t2".to_string()); + let (scan_id_1, scan_expr_id_1) = txn.add_group(scan1, &[]).await.unwrap().ok().unwrap(); + let (scan_id_2, scan_expr_id_2) = txn.add_group(scan2, &[]).await.unwrap().ok().unwrap(); + + assert_eq!( + txn.get_logical_children(scan_id_1).await.unwrap(), + &[scan_expr_id_1] + ); + assert_eq!( + txn.get_logical_children(scan_id_2).await.unwrap(), + &[scan_expr_id_2] + ); + + // Create two join expression that should be in the same group. + let join1 = join(scan_id_1, scan_id_2, "t1.a = t2.b".to_string()); + let join2 = join(scan_id_2, scan_id_1, "t1.a = t2.b".to_string()); + + // Create the group, adding the first expression. + let (join_id, join_expr_id_1) = txn + .add_group(join1, &[scan_id_1, scan_id_2]) + .await + .unwrap() + .ok() + .unwrap(); + // Add the second expression. + let join_expr_id_2 = txn + .add_logical_expression_to_group(join_id, join2, &[scan_id_2, scan_id_1]) + .await + .unwrap() + .ok() + .unwrap(); + + assert_ne!(join_expr_id_1, join_expr_id_2); + assert_eq!( + txn.get_logical_children(join_id).await.unwrap(), + &[join_expr_id_1, join_expr_id_2] + ); + + txn.commit().await.unwrap(); + memo.cleanup().await; +} + +/// Tests a single group merge. See comments in the test itself for more information. +#[ignore] +#[tokio::test] +async fn test_simple_group_link() { + let memo = PersistentMemo::::new().await; + memo.cleanup().await; + + let mut txn = memo.begin().await.unwrap(); + + // Create two scan groups. + let scan1 = scan("t1".to_string()); + let scan2 = scan("t2".to_string()); + let (scan_id_1, _) = txn.add_group(scan1, &[]).await.unwrap().ok().unwrap(); + let (scan_id_2, _) = txn.add_group(scan2, &[]).await.unwrap().ok().unwrap(); + + // Create two join expression that should be in the same group. + // Even though these are obviously the same expression (to humans), the fingerprints will be + // different, and so they will be put into different groups. + let join1 = join(scan_id_1, scan_id_2, "t1.a = t2.b".to_string()); + let join2 = join(scan_id_2, scan_id_1, "t2.b = t1.a".to_string()); + let join_unknown = join2.clone(); + + let (join_group_1, _) = txn + .add_group(join1, &[scan_id_1, scan_id_2]) + .await + .unwrap() + .ok() + .unwrap(); + let (join_group_2, join_expr_2) = txn + .add_group(join2, &[scan_id_2, scan_id_1]) + .await + .unwrap() + .ok() + .unwrap(); + assert_ne!(join_group_1, join_group_2); + + // Assume that some rule was applied to `join1`, and it outputs something like `join_unknown`. + // The txn table will tell us that `join_unknown == join2`. + // Take note here that `join_unknown` is a clone of `join2`, not `join1`. + let (existing_group, not_actually_new_expr_id) = txn + .add_logical_expression_to_group(join_group_1, join_unknown, &[scan_id_2, scan_id_1]) + .await + .unwrap() + .err() + .unwrap(); + assert_eq!(existing_group, join_group_2); + assert_eq!(not_actually_new_expr_id, join_expr_2); + + // The above tells the application that the expression already exists in the txn, specifically + // under `existing_group`. Thus, we should link these two groups together. + txn.merge_groups(join_group_1, join_group_2).await.unwrap(); + + let test_root_1 = txn.get_root_group(join_group_1).await.unwrap(); + let test_root_2 = txn.get_root_group(join_group_2).await.unwrap(); + assert_eq!(test_root_1, test_root_2); + + txn.commit().await.unwrap(); + memo.cleanup().await; +} + +/// Tests merging groups up a chain. +#[ignore] +#[tokio::test] +async fn test_group_merge_ladder() { + let memo = PersistentMemo::::new().await; + memo.cleanup().await; + + let mut txn = memo.begin().await.unwrap(); + + // Build up a tree of true filters that should be collapsed into a single table scan. + let scan_base = scan("t1".to_string()); + let (scan_id, _) = txn.add_group(scan_base, &[]).await.unwrap().ok().unwrap(); + + let filter0 = filter(scan_id, "true".to_string()); + let (filter_id_0, _) = txn + .add_group(filter0, &[scan_id]) + .await + .unwrap() + .ok() + .unwrap(); + + let filter1 = filter(filter_id_0, "true".to_string()); + let (filter_id_1, _) = txn + .add_group(filter1, &[scan_id]) + .await + .unwrap() + .ok() + .unwrap(); + + let filter2 = filter(filter_id_1, "true".to_string()); + let (filter_id_2, _) = txn + .add_group(filter2, &[scan_id]) + .await + .unwrap() + .ok() + .unwrap(); + + let filter3 = filter(filter_id_2, "true".to_string()); + let (filter_id_3, _) = txn + .add_group(filter3, &[scan_id]) + .await + .unwrap() + .ok() + .unwrap(); + + let mut groups = vec![scan_id, filter_id_0, filter_id_1, filter_id_2, filter_id_3]; + + let m0 = txn.merge_groups(filter_id_3, filter_id_2).await.unwrap(); + let m1 = txn.merge_groups(filter_id_2, filter_id_1).await.unwrap(); + let m2 = txn.merge_groups(filter_id_1, filter_id_0).await.unwrap(); + let root = txn.merge_groups(filter_id_0, scan_id).await.unwrap(); + groups.extend_from_slice(&[m0, m1, m2, root]); + + for group_id in groups { + assert_eq!(root, txn.get_root_group(group_id).await.unwrap()); + } + + txn.commit().await.unwrap(); + memo.cleanup().await; +} + +/// Tests merging a bunch of groups together in order to prevent duplicates from being added. +#[ignore] +#[tokio::test] +async fn test_group_merge() { + let memo = PersistentMemo::::new().await; + memo.cleanup().await; + + let mut txn = memo.begin().await.unwrap(); + + // Create a base group. + let scan1 = scan("t1".to_string()); + let (scan_id_1, _) = txn.add_group(scan1, &[]).await.unwrap().ok().unwrap(); + + // Create a bunch of equivalent groups. + let filter0 = filter(scan_id_1, "true".to_string()); + let filter1 = filter(scan_id_1, "1 < 2".to_string()); + let filter2 = filter(scan_id_1, "2 > 1".to_string()); + let filter3 = filter(scan_id_1, "42 != 100".to_string()); + let filter4 = filter(scan_id_1, "10000 > 0".to_string()); + let filter5 = filter(scan_id_1, "1 + 2 = 3".to_string()); + let filter6 = filter(scan_id_1, "true OR false".to_string()); + let filter7 = filter(scan_id_1, "(1 + 1 > -1 AND true) OR false".to_string()); + let (filter_id_0, _) = txn + .add_group(filter0, &[scan_id_1]) + .await + .unwrap() + .ok() + .unwrap(); + let (filter_id_1, _) = txn + .add_group(filter1, &[scan_id_1]) + .await + .unwrap() + .ok() + .unwrap(); + let (filter_id_2, _) = txn + .add_group(filter2, &[scan_id_1]) + .await + .unwrap() + .ok() + .unwrap(); + let (filter_id_3, _) = txn + .add_group(filter3, &[scan_id_1]) + .await + .unwrap() + .ok() + .unwrap(); + let (filter_id_4, _) = txn + .add_group(filter4, &[scan_id_1]) + .await + .unwrap() + .ok() + .unwrap(); + let (filter_id_5, _) = txn + .add_group(filter5, &[scan_id_1]) + .await + .unwrap() + .ok() + .unwrap(); + let (filter_id_6, _) = txn + .add_group(filter6, &[scan_id_1]) + .await + .unwrap() + .ok() + .unwrap(); + let (filter_id_7, _) = txn + .add_group(filter7, &[scan_id_1]) + .await + .unwrap() + .ok() + .unwrap(); + let filters = vec![ + filter_id_0, + filter_id_1, + filter_id_2, + filter_id_3, + filter_id_4, + filter_id_5, + filter_id_6, + filter_id_7, + ]; + + // Merge them all together. + let quarter_0 = txn.merge_groups(filters[0], filters[1]).await.unwrap(); + let quarter_1 = txn.merge_groups(filters[2], filters[3]).await.unwrap(); + let quarter_2 = txn.merge_groups(filters[4], filters[5]).await.unwrap(); + let quarter_3 = txn.merge_groups(filters[6], filters[7]).await.unwrap(); + let semi_0 = txn.merge_groups(quarter_0, quarter_1).await.unwrap(); + let semi_1 = txn.merge_groups(quarter_2, quarter_3).await.unwrap(); + let final_id = txn.merge_groups(semi_0, semi_1).await.unwrap(); + + // Check that the group set is properly representative. + { + let set = txn.get_group_set(final_id).await.unwrap(); + assert_eq!(set.len(), 8); + for id in set { + assert!(filters.contains(&id)); + } + } + + // Create another base group. + let scan2 = scan("t2".to_string()); + let (scan_id_2, _) = txn.add_group(scan2, &[]).await.unwrap().ok().unwrap(); + + // Add a join group. + let join0 = join(filter_id_0, scan_id_2, "t1.a = t2.a".to_string()); + let (join_group_id, join_expr_id) = txn + .add_group(join0, &[filter_id_0, scan_id_2]) + .await + .unwrap() + .ok() + .unwrap(); + + // Adding the duplicate join expressions should return a duplication error containing the IDs of + // the already existing group and expression. + for filter_id in filters { + let join_test = join(filter_id, scan_id_2, "t1.a = t2.a".to_string()); + let (join_group_id_test, join_expr_id_test) = txn + .add_group(join_test, &[filter_id, scan_id_2]) + .await + .unwrap() + .err() + .unwrap(); + assert_eq!(join_group_id, join_group_id_test); + assert_eq!(join_expr_id, join_expr_id_test); + } + + txn.commit().await.unwrap(); + memo.cleanup().await; +} + +/// Tests the exact same scenario as in the "Discovered Duplicates" section in `DESIGN.md`. +#[ignore] +#[tokio::test] +async fn test_cascading_merge() { + let memo = PersistentMemo::::new().await; + memo.cleanup().await; + + let mut txn = memo.begin().await.unwrap(); + + // Create the base groups. + let scan1 = scan("t1".to_string()); + let (g1, _) = txn.add_group(scan1, &[]).await.unwrap().ok().unwrap(); + let scan2 = scan("t2".to_string()); + let (g2, _) = txn.add_group(scan2, &[]).await.unwrap().ok().unwrap(); + + let filter1 = filter(g1, "x > 1000".to_string()); + let (g3, _) = txn.add_group(filter1, &[g1]).await.unwrap().ok().unwrap(); + + // Create two groups that will need to be merged. + let filter2a = filter(g2, "a < 42".to_string()); + let (g4, _) = txn.add_group(filter2a, &[g2]).await.unwrap().ok().unwrap(); + let filter2b = filter(g4, "a < 42 AND 1 = 1".to_string()); + let (g5, _) = txn.add_group(filter2b, &[g4]).await.unwrap().ok().unwrap(); + + // Create groups that are dependent on the to-be-merged groups. + let join1 = join(g3, g4, "t1.x = t2.a".to_string()); + let (g6, _) = txn.add_group(join1, &[g3, g4]).await.unwrap().ok().unwrap(); + let join2 = join(g3, g5, "t1.x = t2.a".to_string()); + let (g7, _) = txn.add_group(join2, &[g3, g5]).await.unwrap().ok().unwrap(); + + // Create more groups that are dependent on the to-be-merged groups. + // TODO actually use a sort expression instead of a `filter` placeholder. + let sort1 = filter(g6, "ORDER BY a".to_string()); + let (g8, _) = txn.add_group(sort1, &[g6]).await.unwrap().ok().unwrap(); + + let sort2 = filter(g7, "ORDER BY a".to_string()); + let (g9, _) = txn.add_group(sort2, &[g7]).await.unwrap().ok().unwrap(); + + // Now that everything is set up, we can merge groups 4 and 5 to begin the cascading process. + let filter_root = txn.merge_groups(g4, g5).await.unwrap(); + assert_eq!(txn.get_root_group(g4).await.unwrap(), filter_root); + assert_eq!(txn.get_root_group(g5).await.unwrap(), filter_root); + + // After merging, the join groups (6 and 7) are technically identical, but we have not merged + // them together yet. However, applying rules will reveal that they are identical, and we will + // know that they need to get merged. + let join1_commute = join(g4, g3, "t1.x = t2.a".to_string()); + let join1_commute_id = txn + .add_logical_expression_to_group(g6, join1_commute, &[g4, g3]) + .await + .unwrap() + .ok() + .unwrap(); + + // Adding this expression should now result in a duplication error and return the above ID. + let join2_commute = join(g5, g3, "t1.x = t2.a".to_string()); + let (existing_g6, existing_id) = txn + .add_logical_expression_to_group(g7, join2_commute, &[g5, g3]) + .await + .unwrap() + .err() + .unwrap(); + assert_eq!(existing_g6, g6); + assert_eq!(existing_id, join1_commute_id); + + // Since the txn table has told us these are duplicates, we can now merge groups 6 and 7. + let join_root = txn.merge_groups(g6, g7).await.unwrap(); + assert_eq!(txn.get_root_group(g6).await.unwrap(), join_root); + assert_eq!(txn.get_root_group(g7).await.unwrap(), join_root); + + // Do a similar thing for the sort groups. We'll skip the expression adding for now and just + // merge them immediately, but remember that the application should observe a duplicate + // somewhere in the txn table before deciding to merge groups. + let sort_root = txn.merge_groups(g8, g9).await.unwrap(); + assert_eq!(txn.get_root_group(g8).await.unwrap(), sort_root); + assert_eq!(txn.get_root_group(g9).await.unwrap(), sort_root); + + txn.commit().await.unwrap(); + memo.cleanup().await; +} diff --git a/optd-mvp/src/migrator/memo/m20241127_000001_fingerprint.rs b/optd-mvp/src/migrator/memo/m20241127_000001_fingerprint.rs new file mode 100644 index 0000000..e153b9e --- /dev/null +++ b/optd-mvp/src/migrator/memo/m20241127_000001_fingerprint.rs @@ -0,0 +1,49 @@ +//! An entity representing a logical expression fingerprint. +//! +//! TODO write docs. + +use crate::migrator::memo::logical_expression::LogicalExpression; +use sea_orm_migration::{prelude::*, schema::*}; + +#[derive(DeriveIden)] +pub enum Fingerprint { + Table, + Id, + LogicalExpressionId, + Kind, + Hash, +} + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(Fingerprint::Table) + .if_not_exists() + .col(pk_auto(Fingerprint::Id)) + .col(integer(Fingerprint::LogicalExpressionId)) + .foreign_key( + ForeignKey::create() + .from(Fingerprint::Table, Fingerprint::LogicalExpressionId) + .to(LogicalExpression::Table, LogicalExpression::Id) + .on_delete(ForeignKeyAction::Cascade) + .on_update(ForeignKeyAction::Cascade), + ) + .col(small_integer(Fingerprint::Kind)) + .col(big_integer(Fingerprint::Hash)) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_table(Table::drop().table(Fingerprint::Table).to_owned()) + .await + } +} diff --git a/optd-mvp/src/migrator/memo/m20241127_000001_group.rs b/optd-mvp/src/migrator/memo/m20241127_000001_group.rs new file mode 100644 index 0000000..6251b6a --- /dev/null +++ b/optd-mvp/src/migrator/memo/m20241127_000001_group.rs @@ -0,0 +1,140 @@ +//! FIXME We aren't really following the cascades framework anymore... +//! +//! An entity representing a group / equivalence class in the Cascades framework. +//! +//! Quoted from the Microsoft article _Extensible query optimizers in practice_: +//! +//! > In the memo, each class of equivalent expressions is called an equivalent class or a group, +//! > and all equivalent expressions within the class are called group expressions or simply +//! > expressions. +//! +//! A Cascades group is defined as a class of equivalent logical or physical expressions. The +//! Cascades framework uses these groups as a way of storing the best query sub-plans for use in the +//! dynamic programming search algorithm. +//! +//! For example, a Cascades group could be the set of expressions containing the logical expressions +//! `Join(A, B)` and `Join(B, A)`, as well as the physical expressions `HashJoin(A, B)` and +//! `NestedLoopJoin(B, A)`. +//! +//! # Columns +//! +//! Each group is assigned a monotonically-increasing (unique) ID. This ID will be important since +//! there are many foreign key references from other tables to `group`. +//! +//! We store an `status` enumeration encoded as an 8-bit integer that is used for quickly +//! determining the state of optimization for this group during the dynamic programming search. +//! +//! We additionally store a `winner` foreign key reference to a physical expression paired with a +//! `cost` foreign key reference to a cost record (FIXME). See the +//! [section](#best-physical-plan-winner) below for more details. +//! +//! Finally, we maintain a union-find graph structure embedded in the group records. See the +//! `DESIGN.md` document for more information. +//! +//! # Entity Relationships +//! +//! ### Child Expressions (Logical and Physical) +//! +//! To retrieve all of a `group`'s equivalent expressions, you must query the +//! [`logical_expression`] or the [`physical_expression`] entities via their foreign keys to +//! `group`. The relationship between [`logical_expression`] and `group` is +//! many-to-one, and the exact same many-to-one relationship is held for [`physical_expression`] to +//! `group`. +//! +//! ### Parent Expressions (Logical and Physical) +//! +//! Additionally, each logical or physical expression can have any number of `group`s as +//! children, and a group can be a child of any expression. Thus, `group` additionally has +//! a many-to-many relationship with [`logical_expression`] and [`physical_expression`] via the +//! [`logical_children`] and [`physical_children`] entities. +//! +//! To reiterate, `group` has **both** a one-to-many **and** a many-to-many relationship +//! with both [`logical_expression`] and [`physical_expression`]. This is due to groups being both +//! parents and children of expressions. +//! +//! ### Best Physical Plan (Winner) +//! +//! The `group` entity also stores a `winner` _nullable_ foreign key reference to +//! a physical expression. This represents the most recent best query plan we have computed. The +//! reason it is nullable is because we may not have come up with any best query plan yet. +//! +//! ### Logical Properties +//! +//! FIXME: Add a logical properties table. +//! +//! Lastly, each `group` record will have a set of logical properties store in the +//! `logical_property` entity, where there is an many-to-one relationship from +//! `logical_property` to `group`. Note that we do not store physical properties directly +//! on the `group`, but rather we store them for each [`physical_expression`] record. +//! +//! [`logical_expression`]: super::logical_expression +//! [`physical_expression`]: super::physical_expression +//! [`logical_children`]: super::logical_children +//! [`physical_children`]: super::physical_children +//! `logical_property`: super::logical_property + +use crate::migrator::memo::physical_expression::PhysicalExpression; +use sea_orm_migration::{prelude::*, schema::*}; + +#[derive(DeriveIden)] +pub enum Group { + Table, + Id, + Status, + Winner, + Cost, + SetSize, + ParentId, + NextId, +} + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(Group::Table) + .if_not_exists() + .col(pk_auto(Group::Id)) + .col(tiny_integer(Group::Status)) + .col(integer_null(Group::Winner)) + .col(big_integer_null(Group::Cost)) + .foreign_key( + ForeignKey::create() + .from(Group::Table, Group::Winner) + .to(PhysicalExpression::Table, PhysicalExpression::Id) + .on_delete(ForeignKeyAction::SetNull) + .on_update(ForeignKeyAction::Cascade), + ) + .col(integer(Group::SetSize)) + .col(integer_null(Group::ParentId)) + .foreign_key( + ForeignKey::create() + .from(Group::Table, Group::ParentId) + .to(Group::Table, Group::Id) + .on_delete(ForeignKeyAction::SetNull) + .on_update(ForeignKeyAction::Cascade), + ) + .col(integer_null(Group::NextId)) + .foreign_key( + ForeignKey::create() + .from(Group::Table, Group::NextId) + .to(Group::Table, Group::Id) + .on_delete(ForeignKeyAction::SetNull) + .on_update(ForeignKeyAction::Cascade), + ) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_table(Table::drop().table(Group::Table).to_owned()) + .await + } +} diff --git a/optd-mvp/src/migrator/memo/m20241127_000001_logical_children.rs b/optd-mvp/src/migrator/memo/m20241127_000001_logical_children.rs new file mode 100644 index 0000000..12b5aa6 --- /dev/null +++ b/optd-mvp/src/migrator/memo/m20241127_000001_logical_children.rs @@ -0,0 +1,65 @@ +//! An entity representing the [`group`] children of every [`logical_expression`]. +//! +//! Formally, this entity is a junction which allows us to represent a many-to-many relationship +//! between [`logical_expression`] and [`group`]. Expressions can have any number of child +//! groups, and every group can be a child of many different expressions, hence the many-to-many +//! relationship. +//! +//! See [`group`] for more details. +//! +//! [`group`]: super::group +//! [`logical_expression`]: super::logical_expression + +use crate::migrator::memo::{group::Group, logical_expression::LogicalExpression}; +use sea_orm_migration::{prelude::*, schema::*}; + +#[derive(DeriveIden)] +pub enum LogicalChildren { + Table, + LogicalExpressionId, + GroupId, +} + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(LogicalChildren::Table) + .if_not_exists() + .col(integer(LogicalChildren::LogicalExpressionId)) + .col(integer(LogicalChildren::GroupId)) + .primary_key( + Index::create() + .col(LogicalChildren::LogicalExpressionId) + .col(LogicalChildren::GroupId), + ) + .foreign_key( + ForeignKey::create() + .from(LogicalChildren::Table, LogicalChildren::LogicalExpressionId) + .to(LogicalExpression::Table, LogicalExpression::Id) + .on_delete(ForeignKeyAction::Cascade) + .on_update(ForeignKeyAction::Cascade), + ) + .foreign_key( + ForeignKey::create() + .from(LogicalChildren::Table, LogicalChildren::GroupId) + .to(Group::Table, Group::Id) + .on_delete(ForeignKeyAction::Cascade) + .on_update(ForeignKeyAction::Cascade), + ) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_table(Table::drop().table(LogicalChildren::Table).to_owned()) + .await + } +} diff --git a/optd-mvp/src/migrator/memo/m20241127_000001_logical_expression.rs b/optd-mvp/src/migrator/memo/m20241127_000001_logical_expression.rs new file mode 100644 index 0000000..9b5eefc --- /dev/null +++ b/optd-mvp/src/migrator/memo/m20241127_000001_logical_expression.rs @@ -0,0 +1,84 @@ +//! An entity representing a logical relational expression. +//! +//! Quoted from the Microsoft article _Extensible query optimizers in practice_: +//! +//! > A logical expression is defined as a tree of logical operators, and corresponds to a +//! > relational algebraic expression. +//! +//! In the Cascades query optimization framework, the memo table stores equivalence classes of +//! expressions (see [`group`]). These equivalence classes, or "groups", store both +//! `logical_expression`s and [`physical_expression`]s. +//! +//! Optimization starts by "exploring" equivalent logical expressions within a group. For example, +//! the logical expressions `Join(A, B)` and `Join(B, A)` are contained in the same group. The +//! logical expressions are defined as a `Join` operator with the groups representing a scan of +//! table `A` and a scan of table `B` as its children. +//! +//! # Columns +//! +//! Each `logical_expression` has a unique primary key ID, but it holds little importance other than +//! helping distinguish between two different expressions. +//! +//! Finally, since there are many different types of operators, we store a variant tag and a data +//! column as JSON to represent the semi-structured data fields of logical operators. +//! +//! # Entity Relationships +//! +//! The main relationship that `logical_expression` has is to [`group`]. It has **both** a +//! one-to-many **and** a many-to-many relationship with [`group`], and you can see more +//! details about this in the module-level documentation for [`group`]. +//! +//! The other relationship that `logical_expression` has is to [`fingerprint`]. This table stores +//! 1 or more fingerprints for every (unique) logical expression. The reason we have multiple +//! fingerprints is that an expression can belong to multiple groups during the exploration phase +//! before the merging of groups. +//! +//! [`group`]: super::group +//! [`physical_expression`]: super::physical_expression +//! [`fingerprint`]: super::fingerprint + +use crate::migrator::memo::group::Group; +use sea_orm_migration::{prelude::*, schema::*}; + +#[derive(DeriveIden)] +pub enum LogicalExpression { + Table, + Id, + GroupId, + Kind, + Data, +} + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(LogicalExpression::Table) + .if_not_exists() + .col(pk_auto(LogicalExpression::Id)) + .col(integer(LogicalExpression::GroupId)) + .foreign_key( + ForeignKey::create() + .from(LogicalExpression::Table, LogicalExpression::GroupId) + .to(Group::Table, Group::Id) + .on_delete(ForeignKeyAction::Cascade) + .on_update(ForeignKeyAction::Cascade), + ) + .col(small_integer(LogicalExpression::Kind)) + .col(json(LogicalExpression::Data)) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_table(Table::drop().table(LogicalExpression::Table).to_owned()) + .await + } +} diff --git a/optd-mvp/src/migrator/memo/m20241127_000001_physical_children.rs b/optd-mvp/src/migrator/memo/m20241127_000001_physical_children.rs new file mode 100644 index 0000000..06ce259 --- /dev/null +++ b/optd-mvp/src/migrator/memo/m20241127_000001_physical_children.rs @@ -0,0 +1,68 @@ +//! An entity representing the [`group`] children of every [`physical_expression`]. +//! +//! Formally, this entity is a junction which allows us to represent a many-to-many relationship +//! between [`physical_expression`] and [`group`]. Expressions can have any number of child +//! groups, and every group can be a child of many different expressions, hence the many-to-many +//! relationship. +//! +//! See [`group`] for more details. +//! +//! [`group`]: super::group +//! [`physical_expression`]: super::physical_expression + +use crate::migrator::memo::{group::Group, physical_expression::PhysicalExpression}; +use sea_orm_migration::{prelude::*, schema::*}; + +#[derive(DeriveIden)] +pub enum PhysicalChildren { + Table, + PhysicalExpressionId, + GroupId, +} + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(PhysicalChildren::Table) + .if_not_exists() + .col(integer(PhysicalChildren::PhysicalExpressionId)) + .col(integer(PhysicalChildren::GroupId)) + .primary_key( + Index::create() + .col(PhysicalChildren::PhysicalExpressionId) + .col(PhysicalChildren::GroupId), + ) + .foreign_key( + ForeignKey::create() + .from( + PhysicalChildren::Table, + PhysicalChildren::PhysicalExpressionId, + ) + .to(PhysicalExpression::Table, PhysicalExpression::Id) + .on_delete(ForeignKeyAction::Cascade) + .on_update(ForeignKeyAction::Cascade), + ) + .foreign_key( + ForeignKey::create() + .from(PhysicalChildren::Table, PhysicalChildren::GroupId) + .to(Group::Table, Group::Id) + .on_delete(ForeignKeyAction::Cascade) + .on_update(ForeignKeyAction::Cascade), + ) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_table(Table::drop().table(PhysicalChildren::Table).to_owned()) + .await + } +} diff --git a/optd-mvp/src/migrator/memo/m20241127_000001_physical_expression.rs b/optd-mvp/src/migrator/memo/m20241127_000001_physical_expression.rs new file mode 100644 index 0000000..88cd63b --- /dev/null +++ b/optd-mvp/src/migrator/memo/m20241127_000001_physical_expression.rs @@ -0,0 +1,84 @@ +//! An entity representing a physical plan expression. +//! +//! Quoted from the Microsoft article _Extensible query optimizers in practice_: +//! +//! > A physical expression is a tree of physical operators, which is also referred to as the +//! > _physical plan_ or simply _plan_. +//! +//! In the Cascades query optimization framework, the memo table stores equivalence classes of +//! expressions (see [`group`]). These equivalence classes, or "groups", store both +//! [`logical_expression`]s and `physical_expression`s. +//! +//! Optimization starts by exploring equivalent logical expressions within a group, and then it +//! proceeds to implement / optimize those logical operators into physical operators. For example, +//! the logical expression `Join(A, B)` could be implemented into a `HashJoin(A, B)` or a +//! `NestedLoopJoin(A, B)`, and both of these new physical expressions would be contained in the +//! same group. +//! +//! # Columns +//! +//! Each `physical_expression` has a unique primary key ID, and other tables will store a foreign +//! key reference to a specific `physical_expression`s. +//! +//! Note that `physical_expression` does **not** store a fingerprint. Remember that we want to +//! detect duplicates in the logical exploration phase. If there are no duplicate logical +//! expressions in the memo table, then there cannot be any duplicate physical expressions, which +//! are derived from said deduplicated logical expressions. +//! +//! Finally, since there are many different types of operators, we store a variant tag and a data +//! column as JSON to represent the semi-structured data fields of logical operators. +//! +//! # Entity Relationships +//! +//! The only relationship that `physical_expression` has is to [`group`]. It has **both** a +//! one-to-many **and** a many-to-many relationship with [`group`], and you can see more +//! details about this in the module-level documentation for [`group`]. +//! +//! [`group`]: super::group +//! [`logical_expression`]: super::logical_expression + +use crate::migrator::memo::group::Group; +use sea_orm_migration::{prelude::*, schema::*}; + +#[derive(DeriveIden)] +pub enum PhysicalExpression { + Table, + Id, + GroupId, + Kind, + Data, +} + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(PhysicalExpression::Table) + .if_not_exists() + .col(pk_auto(PhysicalExpression::Id)) + .col(integer(PhysicalExpression::GroupId)) + .foreign_key( + ForeignKey::create() + .from(PhysicalExpression::Table, PhysicalExpression::GroupId) + .to(Group::Table, Group::Id) + .on_delete(ForeignKeyAction::Cascade) + .on_update(ForeignKeyAction::Cascade), + ) + .col(small_integer(PhysicalExpression::Kind)) + .col(json(PhysicalExpression::Data)) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_table(Table::drop().table(PhysicalExpression::Table).to_owned()) + .await + } +} diff --git a/optd-mvp/src/migrator/memo/mod.rs b/optd-mvp/src/migrator/memo/mod.rs new file mode 100644 index 0000000..a419189 --- /dev/null +++ b/optd-mvp/src/migrator/memo/mod.rs @@ -0,0 +1,16 @@ +//! Entities related to the memo table used for dynamic programming in the our query optimization +//! framework. + +pub(crate) mod m20241127_000001_fingerprint; +pub(crate) mod m20241127_000001_group; +pub(crate) mod m20241127_000001_logical_children; +pub(crate) mod m20241127_000001_logical_expression; +pub(crate) mod m20241127_000001_physical_children; +pub(crate) mod m20241127_000001_physical_expression; + +pub(crate) use m20241127_000001_fingerprint as fingerprint; +pub(crate) use m20241127_000001_group as group; +pub(crate) use m20241127_000001_logical_children as logical_children; +pub(crate) use m20241127_000001_logical_expression as logical_expression; +pub(crate) use m20241127_000001_physical_children as physical_children; +pub(crate) use m20241127_000001_physical_expression as physical_expression; diff --git a/optd-mvp/src/migrator/mod.rs b/optd-mvp/src/migrator/mod.rs new file mode 100644 index 0000000..92f100a --- /dev/null +++ b/optd-mvp/src/migrator/mod.rs @@ -0,0 +1,32 @@ +//! This module defines the tables and their schemas for representing a persistent memo table. +//! +//! The most important tables represented here are the [`group`], [`logical_expression`], and +//! [`physical_expression`] tables. See the corresponding modules for more information on their +//! relations and fields. +//! +//! See the SeaORM docs for more information specific to migrations. +//! +//! [`group`]: memo::group +//! [`logical_expression`]: memo::logical_expression +//! [`physical_expression`]: memo::physical_expression + +use sea_orm_migration::prelude::*; + +mod memo; + +/// A unit struct that implements the [`MigratorTrait`] for running custom migrations. +pub struct Migrator; + +#[async_trait::async_trait] +impl MigratorTrait for Migrator { + fn migrations() -> Vec> { + vec![ + Box::new(memo::group::Migration), + Box::new(memo::fingerprint::Migration), + Box::new(memo::logical_expression::Migration), + Box::new(memo::logical_children::Migration), + Box::new(memo::physical_expression::Migration), + Box::new(memo::physical_children::Migration), + ] + } +}