chore: offline builds with submodules (#486)

right now, we are cloning libpg_query on every build if its not
available locally. this is not only unnecessary, it also makes
reproducible builds even harder. this reverts that back to using
submodules instead.

also unified the directory structures across crates and removed the code
that was intended to support multi-version builds. I think its
unnecessary to even support that in our case since postgres syntax is
backward compatible, and `EXPLAIN` exists since 6.4.

Author: psteinroe

.gitmodules

@@ -2,3 +2,7 @@
 	path = lib/tree_sitter_sql/tree-sitter-sql
 	url = https://github.com/DerekStride/tree-sitter-sql
 	branch = gh-pages
+[submodule "crates/pgt_query/vendor/libpg_query"]
+	path = crates/pgt_query/vendor/libpg_query
+	url = https://github.com/pganalyze/libpg_query.git
+	branch = 17-latest

Cargo.lock

@@ -3,39 +3,34 @@ use std::fs;
 use std::io::Write;
 use std::path::PathBuf;
 
-// TODO make this selectable via feature flags
-static LIBPG_QUERY_TAG: &str = "17-6.1.0";
+static LIBPG_QUERY_TAG: &str = "17-latest";
 
-/// Downloads the `kwlist.h` file from the specified version of `libpg_query`
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let version = LIBPG_QUERY_TAG.to_string();
-
-    // Check for the postgres header file in the source tree first
     let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?);
-    let headers_dir = manifest_dir.join("postgres").join(&version);
-    let kwlist_path = headers_dir.join("kwlist.h");
+    let vendor_dir = manifest_dir.join("vendor").join(LIBPG_QUERY_TAG);
+    let kwlist_path = vendor_dir.join("kwlist.h");
 
-    // Only download if the file doesn't exist
+    // Download kwlist.h if not already present in source directory
     if !kwlist_path.exists() {
         println!(
-            "cargo:warning=Downloading kwlist.h for libpg_query {}",
-            version
+            "cargo:warning=Downloading kwlist.h for libpg_query {} to source directory",
+            LIBPG_QUERY_TAG
         );
 
-        fs::create_dir_all(&headers_dir)?;
+        fs::create_dir_all(&vendor_dir)?;
 
-        let proto_url = format!(
+        let kwlist_url = format!(
             "https://raw.githubusercontent.com/pganalyze/libpg_query/{}/src/postgres/include/parser/kwlist.h",
-            version
+            LIBPG_QUERY_TAG
         );
 
-        let response = ureq::get(&proto_url).call()?;
+        let response = ureq::get(&kwlist_url).call()?;
         let content = response.into_string()?;
 
         let mut file = fs::File::create(&kwlist_path)?;
         file.write_all(content.as_bytes())?;
 
-        println!("cargo:warning=Successfully downloaded kwlist.h");
+        println!("cargo:warning=Successfully downloaded kwlist.h to source");
     }
 
     println!(

crates/pgt_lexer_codegen/build.rs

@@ -18,10 +18,7 @@ pgt_query_macros = { workspace = true }
 
 
 [features]
-default     = ["postgres-17"]
-postgres-15 = []
-postgres-16 = []
-postgres-17 = []
+default = []
 
 [build-dependencies]
 bindgen     = "0.72.0"

crates/pgt_lexer_codegen/postgres/17-6.1.0/kwlist.h renamed to crates/pgt_lexer_codegen/vendor/17-latest/kwlist.h

@@ -1,80 +1,48 @@
-#![cfg_attr(feature = "clippy", feature(plugin))]
-#![cfg_attr(feature = "clippy", plugin(clippy))]
-
 use fs_extra::dir::CopyOptions;
 use glob::glob;
 use std::env;
 use std::path::PathBuf;
 use std::process::Command;
 
 static LIBRARY_NAME: &str = "pg_query";
-static LIBPG_QUERY_REPO: &str = "https://github.com/pganalyze/libpg_query.git";
-fn get_libpg_query_tag() -> &'static str {
-    #[cfg(feature = "postgres-15")]
-    return "15-5.3.0";
-    #[cfg(feature = "postgres-16")]
-    return "16-6.1.0";
-    #[cfg(feature = "postgres-17")]
-    return "17-6.1.0";
-}
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let libpg_query_tag = get_libpg_query_tag();
     let out_dir = PathBuf::from(env::var("OUT_DIR")?);
-    let vendor_dir = out_dir.join("vendor");
-    let libpg_query_dir = vendor_dir.join("libpg_query").join(libpg_query_tag);
-    let stamp_file = libpg_query_dir.join(".stamp");
+    let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?);
+    let libpg_query_submodule = manifest_dir.join("vendor").join("libpg_query");
 
-    let src_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?).join("src");
+    let src_dir = manifest_dir.join("src");
     let target = env::var("TARGET").unwrap();
     let is_emscripten = target.contains("emscripten");
 
-    // Configure cargo through stdout
     println!("cargo:rustc-link-search=native={}", out_dir.display());
     println!("cargo:rustc-link-lib=static={LIBRARY_NAME}");
 
-    // Clone libpg_query if not already present
-    if !stamp_file.exists() {
-        println!("cargo:warning=Cloning libpg_query {}", libpg_query_tag);
-
-        // Create vendor directory
-        std::fs::create_dir_all(&vendor_dir)?;
-
-        // Clone the repository with partial clone for faster download
-        let status = Command::new("git")
-            .args([
-                "clone",
-                "--filter=blob:none",
-                "--depth",
-                "1",
-                "--branch",
-                libpg_query_tag,
-                LIBPG_QUERY_REPO,
-                libpg_query_dir.to_str().unwrap(),
-            ])
-            .status()?;
-
-        if !status.success() {
-            return Err("Failed to clone libpg_query".into());
-        }
-
-        // Create stamp file
-        std::fs::File::create(&stamp_file)?;
+    // Check if submodule exists
+    if !libpg_query_submodule.join(".git").exists() && !libpg_query_submodule.join("src").exists() {
+        return Err(
+            "libpg_query submodule not found. Please run: git submodule update --init --recursive"
+                .into(),
+        );
     }
 
-    // Tell cargo to rerun if the stamp file is deleted
-    println!("cargo:rerun-if-changed={}", stamp_file.display());
+    // Tell cargo to rerun if the submodule changes
+    println!(
+        "cargo:rerun-if-changed={}",
+        libpg_query_submodule.join("src").display()
+    );
 
-    // Copy necessary files to OUT_DIR for compilation
+    // copy necessary files to out_dir for compilation
     let out_header_path = out_dir.join(LIBRARY_NAME).with_extension("h");
     let out_protobuf_path = out_dir.join("protobuf");
 
     let source_paths = vec![
-        libpg_query_dir.join(LIBRARY_NAME).with_extension("h"),
-        libpg_query_dir.join("Makefile"),
-        libpg_query_dir.join("src"),
-        libpg_query_dir.join("protobuf"),
-        libpg_query_dir.join("vendor"),
+        libpg_query_submodule.join(LIBRARY_NAME).with_extension("h"),
+        libpg_query_submodule.join("postgres_deparse.h"),
+        libpg_query_submodule.join("Makefile"),
+        libpg_query_submodule.join("src"),
+        libpg_query_submodule.join("protobuf"),
+        libpg_query_submodule.join("vendor"),
     ];
 
     let copy_options = CopyOptions {
@@ -84,17 +52,17 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     fs_extra::copy_items(&source_paths, &out_dir, &copy_options)?;
 
-    // Compile the C library.
+    // compile the c library.
     let mut build = cc::Build::new();
 
-    // Configure for Emscripten if needed
+    // configure for emscripten if needed
     if is_emscripten {
-        // Use emcc as the compiler instead of gcc/clang
+        // use emcc as the compiler instead of gcc/clang
         build.compiler("emcc");
-        // Use emar as the archiver instead of ar
+        // use emar as the archiver instead of ar
         build.archiver("emar");
-        // Note: We don't add WASM-specific flags here as this creates a static library
-        // The final linking flags should be added when building the final WASM module
+        // note: we don't add wasm-specific flags here as this creates a static library
+        // the final linking flags should be added when building the final wasm module
     }
 
     build
@@ -115,7 +83,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .include(out_dir.join("./vendor"))
         .include(out_dir.join("./src/postgres/include"))
         .include(out_dir.join("./src/include"))
-        .warnings(false); // Avoid unnecessary warnings, as they are already considered as part of libpg_query development
+        .warnings(false); // avoid unnecessary warnings, as they are already considered as part of libpg_query development
     if env::var("PROFILE").unwrap() == "debug" || env::var("DEBUG").unwrap() == "1" {
         build.define("USE_ASSERT_CHECKING", None);
     }

crates/pgt_query/Cargo.toml

@@ -3,47 +3,34 @@ use std::fs;
 use std::io::Write;
 use std::path::PathBuf;
 
-// This should match the version used by pgt_query crate
-// You can configure this via environment variable PG_QUERY_VERSION if needed
-static LIBPG_QUERY_TAG: &str = "17-6.1.0";
+static LIBPG_QUERY_TAG: &str = "17-latest";
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Allow version override via environment variable
-    let version = env::var("PG_QUERY_VERSION").unwrap_or_else(|_| LIBPG_QUERY_TAG.to_string());
-
-    // Get the manifest directory (source directory)
     let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?);
-    let postgres_dir = manifest_dir.join("postgres");
-    let proto_filename = format!("{}.proto", version);
-    let proto_path = postgres_dir.join(&proto_filename);
+    let vendor_dir = manifest_dir.join("vendor").join(LIBPG_QUERY_TAG);
+    let proto_path = vendor_dir.join("pg_query.proto");
 
     // Download proto file if not already present in source directory
     if !proto_path.exists() {
         println!(
             "cargo:warning=Downloading pg_query.proto for libpg_query {} to source directory",
-            version
+            LIBPG_QUERY_TAG
         );
 
-        // Create postgres directory if it doesn't exist
-        fs::create_dir_all(&postgres_dir)?;
+        fs::create_dir_all(&vendor_dir)?;
 
-        // Download the proto file
         let proto_url = format!(
             "https://raw.githubusercontent.com/pganalyze/libpg_query/{}/protobuf/pg_query.proto",
-            version
+            LIBPG_QUERY_TAG
         );
 
         let response = ureq::get(&proto_url).call()?;
         let proto_content = response.into_string()?;
 
-        // Write proto file to source directory
         let mut file = fs::File::create(&proto_path)?;
         file.write_all(proto_content.as_bytes())?;
 
-        println!(
-            "cargo:warning=Successfully downloaded pg_query.proto to {}",
-            proto_path.display()
-        );
+        println!("cargo:warning=Successfully downloaded pg_query.proto to source");
     }
 
     // Set environment variable for the proc macro