Type the lazy __index__ consistently with the in-memory writer

writeDuckDBTableParquet()/buildTableSelectSQL() emitted the __index__ column as
an uncast (offset + row_number()), i.e. always BIGINT, while the in-memory
writer narrows the index (or widens it via index_max). So the same resource got
a different __index__ type depending on which path wrote it, and a cross-path
append (in-memory int32 part 0 + lazy BIGINT part 1) produced a
schema-inconsistent, unreadable resource.

The lazy path now chooses the index integer type up front and CASTs it in SQL,
matching the in-memory writer: narrow by range on a fresh write, honor index_max
when supplied, and pin to part 0's on-disk type on append (read via
readParquetSchema). index_max is threaded through the existing ... to
writeDuckDBTableParquet; row count is already computed on this path, so there is
no extra scan. Adds a .duckdbIntTypeName() Arrow->DuckDB type-name helper.

Tests cover lazy narrowing, index_max = Inf -> int64, and append pinning to
part 0. No behavior change for readers; small resources now narrow on the lazy
path as they already did in-memory (BIGINT -> smallest int).

Bump version to 0.99.5; update NEWS.

Author: aboyounp-gne

DESCRIPTION

@@ -1,5 +1,5 @@
 Package: DuckDBDataFrame
-Version: 0.99.4
+Version: 0.99.5
 Date: 2026-07-22
 Title: DuckDB-Backed DataFrame and Table Structures
 Description:

NEWS.md

@@ -1,3 +1,17 @@
+# DuckDBDataFrame 0.99.5
+
+## Bug fixes
+
+- The lazy SQL write path (`writeDuckDBTableParquet()` / `buildTableSelectSQL()`)
+  now types the `__index__` column the same way the in-memory writer does,
+  instead of always emitting a BIGINT `row_number()`. It `CAST`s the index to a
+  type chosen by range (narrowed on a fresh write, `index_max` honored, or pinned
+  to part 0's on-disk type on append), so a resource written or appended across
+  both write paths keeps one consistent `__index__` type. Previously an
+  in-memory part 0 (narrowed) plus a lazy append (BIGINT) produced a
+  schema-inconsistent, unreadable resource, and the same table had a different
+  index type depending on which path wrote it.
+
 # DuckDBDataFrame 0.99.4
 
 ## Bug fixes

R/parquet-io.R

@@ -420,11 +420,25 @@ function(query_sql, target_path, order_cols = NULL, partition_by = NULL,
     }, character(1L))
 }
 
+.duckdbIntTypeName <- function(arrow_type) {
+    switch(arrowTypeToName(arrow_type),
+           "int8"   = "TINYINT",
+           "int16"  = "SMALLINT",
+           "int32"  = "INTEGER",
+           "int64"  = "BIGINT",
+           "uint8"  = "UTINYINT",
+           "uint16" = "USMALLINT",
+           "uint32" = "UINTEGER",
+           "uint64" = "UBIGINT",
+           stop("unsupported __index__ integer type: ",
+                arrowTypeToName(arrow_type)))
+}
+
 #' @importFrom dbplyr sql_render
 #' @importFrom DBI dbQuoteIdentifier
 buildTableSelectSQL <-
 function(x, indexcol = NULL, keycol = NULL, dimtbl = NULL, offset = 0L,
-         conn = dbconn(x))
+         index_type = NULL, conn = dbconn(x))
 {
     if (!inherits(x, "DuckDBTable"))
         stop("'x' must be a DuckDBTable")
@@ -438,9 +452,11 @@ function(x, indexcol = NULL, keycol = NULL, dimtbl = NULL, offset = 0L,
 
     if (!is.null(indexcol)) {
         qidx <- as.character(dbQuoteIdentifier(conn, indexcol))
-        select_parts <- c(select_parts,
-                          sprintf("(%s + row_number() OVER (ORDER BY (SELECT 1))) AS %s",
-                                  format(offset, scientific = FALSE, trim = TRUE), qidx))
+        idx_expr <- sprintf("(%s + row_number() OVER (ORDER BY (SELECT 1)))",
+                            format(offset, scientific = FALSE, trim = TRUE))
+        if (!is.null(index_type))
+            idx_expr <- sprintf("CAST(%s AS %s)", idx_expr, index_type)
+        select_parts <- c(select_parts, sprintf("%s AS %s", idx_expr, qidx))
         output_names <- c(output_names, indexcol)
     }
 
@@ -501,7 +517,7 @@ function(x, indexcol = NULL, keycol = NULL, dimtbl = NULL, offset = 0L,
 writeDuckDBTableParquet <-
 function(x, path, indexcol = "__index__", keycol = "__name__", dimtbl = NULL,
          append = FALSE, offset = 0L, part = NULL, part_digits = 0L,
-         cluster_by = NULL, ...)
+         cluster_by = NULL, index_max = NULL, ...)
 {
     if (!inherits(x, "DuckDBTable"))
         stop("'x' must be a DuckDBTable")
@@ -513,9 +529,24 @@ function(x, path, indexcol = "__index__", keycol = "__name__", dimtbl = NULL,
         create = TRUE)
 
     conn <- dbconn(x)
+
+    n <- nrow(x)
+    index_type <- NULL
+    if (!is.null(indexcol)) {
+        idx_arrow <- if (isTRUE(append)) {
+            readParquetSchema(prep$path,
+                              columns = indexcol)$GetFieldByName(indexcol)$type
+        } else if (!is.null(index_max)) {
+            arrowIntType(c(0, index_max))
+        } else {
+            arrowIntType(c(0, prep$offset + n))
+        }
+        index_type <- .duckdbIntTypeName(idx_arrow)
+    }
+
     built <- buildTableSelectSQL(x, indexcol = indexcol, keycol = keycol,
                                  dimtbl = dimtbl, offset = prep$offset,
-                                 conn = conn)
+                                 index_type = index_type, conn = conn)
     # A clustering key (cluster_by) chooses the physical row order to make
     # row-group zonemaps prune range queries on its columns.
     spec <- .asClusterSpec(cluster_by)
@@ -530,7 +561,6 @@ function(x, path, indexcol = "__index__", keycol = "__name__", dimtbl = NULL,
                                     order_cols = order_cols)
     DBI::dbExecute(conn, copy_sql)
 
-    n <- nrow(x)
     sample_n <- min(100L, max(1L, n))
     sample_df <- as.data.frame(arrow::read_parquet(prep$pq_path))
     if (nrow(sample_df) > sample_n)

man/parquet-io.Rd

@@ -424,3 +424,47 @@ test_that("reading wide numeric types warns about possible precision loss", {
     expect_no_warning(DuckDBDataFrame(tf2, datacols = "d", keycol = "id"),
                       message = "precision")
 })
+
+.f1IndexType <- function(dir) {
+    fs <- sort(list.files(dir, pattern = "parquet$", recursive = TRUE,
+                          full.names = TRUE))
+    vapply(fs, function(p)
+        arrow::ParquetFileReader$create(p)$GetSchema()$
+            GetFieldByName("__index__")$type$ToString(), character(1L))
+}
+
+test_that("lazy writeParquet narrows __index__ and honors index_max", {
+    skip_if_not_installed("arrow")
+    src <- tempfile(fileext = ".parquet"); on.exit(unlink(src), add = TRUE)
+    arrow::write_parquet(data.frame(v = 1:5), src)
+    ddf <- DuckDBDataFrame(src)
+
+    # Small lazy write narrows (uint8), no longer defaults to int64/BIGINT.
+    out <- tempfile()
+    writeDuckDBTableParquet(ddf, out, indexcol = "__index__", keycol = NULL)
+    expect_identical(unname(.f1IndexType(out)), "uint8")
+
+    # index_max = Inf forces int64 (parity with the in-memory writer).
+    out2 <- tempfile()
+    writeDuckDBTableParquet(ddf, out2, indexcol = "__index__", keycol = NULL,
+                            index_max = Inf)
+    expect_identical(unname(.f1IndexType(out2)), "int64")
+})
+
+test_that("lazy append pins __index__ to part 0's type (schema-consistent parts)", {
+    skip_if_not_installed("arrow")
+    src <- tempfile(fileext = ".parquet"); on.exit(unlink(src), add = TRUE)
+    arrow::write_parquet(data.frame(v = 1:5), src)
+    ddf <- DuckDBDataFrame(src)
+
+    dir <- tempfile()
+    writeDuckDBTableParquet(ddf, dir, indexcol = "__index__", keycol = NULL,
+                            part = 0L, part_digits = 2L, append = FALSE,
+                            index_max = Inf)                      # part 0 int64
+    writeDuckDBTableParquet(ddf, dir, indexcol = "__index__", keycol = NULL,
+                            offset = 5, part = 1L, part_digits = 2L,
+                            append = TRUE)                        # no index_max
+    types <- .f1IndexType(dir)
+    expect_length(types, 2L)
+    expect_true(all(types == "int64"))    # append pinned to part 0, not narrowed
+})