diff --git a/.crow/build-all-versions-install-deps-amd64.yaml b/.crow/build-all-versions-install-deps-amd64.yaml index b1f33e4..285a4e2 100644 --- a/.crow/build-all-versions-install-deps-amd64.yaml +++ b/.crow/build-all-versions-install-deps-amd64.yaml @@ -10,7 +10,7 @@ labels: steps: - name: 'Install deps and bincraft' - image: "reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}" + image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' pull: true environment: REPO_RO_TOKEN: diff --git a/.crow/build-all-versions-install-deps-arm64.yaml b/.crow/build-all-versions-install-deps-arm64.yaml index 0ef90e3..98decea 100644 --- a/.crow/build-all-versions-install-deps-arm64.yaml +++ b/.crow/build-all-versions-install-deps-arm64.yaml @@ -9,7 +9,7 @@ labels: steps: - name: 'Install deps and bincraft' - image: "reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}" + image: 'reg.devxy.io/rpkgs/build-env-${OS}:${OS_VERSION}' pull: true environment: REPO_RO_TOKEN: diff --git a/local/build-all.R b/local/build-all.R index 8c181f3..4e7b44e 100644 --- a/local/build-all.R +++ b/local/build-all.R @@ -21,8 +21,14 @@ pkgs <- if (sensitive_only) { readRDS("/mnt/cache/packages/pkgs_to_build.rds") } # Back-compat: tolerate an older RDS without the column (treat all as non-sensitive) -if (is.null(pkgs$r_minor_sensitive)) pkgs$r_minor_sensitive <- FALSE -sprintf("Total# of remaining package versions: %s (sensitive_only=%s)", nrow(pkgs), sensitive_only) +if (is.null(pkgs$r_minor_sensitive)) { + pkgs$r_minor_sensitive <- FALSE +} +sprintf( + "Total# of remaining package versions: %s (sensitive_only=%s)", + nrow(pkgs), + sensitive_only +) # Split into chunks for this worker chunks <- split(pkgs, cut(seq_len(nrow(pkgs)), split_into, labels = FALSE)) @@ -40,27 +46,33 @@ s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds") sprintf("S3 cache: %s files", length(s3_cache)) n <- nrow(chunk) -mapply(function(pkg, ver, sens, i) { - cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens)) - bincraft::build_binary_package( - pkg, - tag = ver, - is_r_minor_sensitive = isTRUE(sens), - s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", - s3_region = "eu-central-003", - s3_bucket = "devxy-rpkgs-binaries", - s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), - s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), - s3_package_cache = s3_cache, - metadata_db_host = "r-binaries.devxy.io", - metadata_db_name = "build_metadata", - metadata_db_table = "single_builds", - metadata_db_user = "rpkgs", - metadata_db_password = Sys.getenv("PGPASS"), - metadata_db_sslmode = "require", - metadata_db_port = 15432, - archive = TRUE, - upload = TRUE, - store_build_metadata = TRUE - ) -}, chunk$Package, chunk$Version, chunk$r_minor_sensitive, seq_len(n)) +mapply( + function(pkg, ver, sens, i) { + cat(sprintf("[%d/%d] %s_%s (r_minor_sensitive=%s)\n", i, n, pkg, ver, sens)) + bincraft::build_binary_package( + pkg, + tag = ver, + is_r_minor_sensitive = isTRUE(sens), + s3_endpoint = "https://s3.eu-central-003.backblazeb2.com", + s3_region = "eu-central-003", + s3_bucket = "devxy-rpkgs-binaries", + s3_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"), + s3_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"), + s3_package_cache = s3_cache, + metadata_db_host = "r-binaries.devxy.io", + metadata_db_name = "build_metadata", + metadata_db_table = "single_builds", + metadata_db_user = "rpkgs", + metadata_db_password = Sys.getenv("PGPASS"), + metadata_db_sslmode = "require", + metadata_db_port = 15432, + archive = TRUE, + upload = TRUE, + store_build_metadata = TRUE + ) + }, + chunk$Package, + chunk$Version, + chunk$r_minor_sensitive, + seq_len(n) +) diff --git a/local/packages-to-build.R b/local/packages-to-build.R index 3b23276..7a3652d 100644 --- a/local/packages-to-build.R +++ b/local/packages-to-build.R @@ -17,7 +17,11 @@ suppressPackageStartupMessages(library(data.table)) arch = Sys.getenv("ARCH") # target: alpine-322, ubuntu-2404, redhat-9, etc. -platform = paste(Sys.getenv("OS"), gsub("[.]", "", Sys.getenv("OS_VERSION")), sep = "-") +platform = paste( + Sys.getenv("OS"), + gsub("[.]", "", Sys.getenv("OS_VERSION")), + sep = "-" +) # Use bincraft's codename detection for S3 paths (e.g. "rhel10" not "redhat10") codename = bincraft::set_codename(NULL) @@ -34,7 +38,9 @@ con <- DBI::dbConnect( cran_archive = tools::CRAN_archive_db() cran_release = tools::CRAN_package_db() # Subset cran_archive to only those packages -cran_archive_in_release <- cran_archive[names(cran_archive) %in% cran_release$Package] +cran_archive_in_release <- cran_archive[ + names(cran_archive) %in% cran_release$Package +] archive_versions <- rbindlist( lapply(names(cran_archive), function(pkg) { @@ -126,11 +132,15 @@ query_error <- function(pkg, ver) { # Fetch all relevant columns from the database errored_pkgs <- DBI::dbGetQuery( con, - sprintf("SELECT name, tag FROM single_builds WHERE error_occurred = TRUE and platform='%s' and arch='%s'", platform, arch) + sprintf( + "SELECT name, tag FROM single_builds WHERE error_occurred = TRUE and platform='%s' and arch='%s'", + platform, + arch + ) ) errored_pkgs <- as.data.table(errored_pkgs) setkey(pkgs_to_build, Package, Version) -setnames(errored_pkgs, c("Package","Version")) +setnames(errored_pkgs, c("Package", "Version")) setkey(errored_pkgs, Package, Version) ### Final subsetting @@ -152,12 +162,14 @@ release_meta <- data.table( ) meta <- release_meta[Package %in% unique(pkgs$Package)] -meta[, triage := mapply( - classify_from_metadata, - NeedsCompilation, - LinkingTo, - MoreArgs = list(risky_deps = risky_deps) -)] +meta[, + triage := mapply( + classify_from_metadata, + NeedsCompilation, + LinkingTo, + MoreArgs = list(risky_deps = risky_deps) + ) +] # Only the "ambiguous" compiled packages need a source grep. ambiguous <- meta[triage == "ambiguous", Package] @@ -165,19 +177,35 @@ sensitive_ambiguous <- character() if (length(ambiguous) > 0L) { tmp_src <- file.path(tempdir(), "abi_src") dir.create(tmp_src, showWarnings = FALSE, recursive = TRUE) - sens <- vapply(ambiguous, function(pkg) { - out <- tryCatch({ - dl <- utils::download.packages( - pkg, destdir = tmp_src, - repos = "https://cloud.r-project.org", quiet = TRUE - ) - isTRUE(as.logical(bincraft::needs_per_minor_recompile(dl[1L, 2L]))) - }, error = function(e) TRUE) # fail safe: treat as sensitive - out - }, logical(1L)) + sens <- vapply( + ambiguous, + function(pkg) { + out <- tryCatch( + { + dl <- utils::download.packages( + pkg, + destdir = tmp_src, + repos = "https://cloud.r-project.org", + quiet = TRUE + ) + isTRUE(as.logical(bincraft::needs_per_minor_recompile(dl[1L, 2L]))) + }, + error = function(e) TRUE + ) # fail safe: treat as sensitive + out + }, + logical(1L) + ) sensitive_ambiguous <- ambiguous[sens] } -sensitive_pkgs <- unique(c(meta[triage == "sensitive", Package], sensitive_ambiguous)) +sensitive_pkgs <- unique(c( + meta[triage == "sensitive", Package], + sensitive_ambiguous +)) pkgs[, r_minor_sensitive := Package %in% sensitive_pkgs] -sprintf("R-minor-sensitive packages: %s of %s", length(sensitive_pkgs), uniqueN(pkgs$Package)) +sprintf( + "R-minor-sensitive packages: %s of %s", + length(sensitive_pkgs), + uniqueN(pkgs$Package) +) diff --git a/local/tests/test-r-minor-helpers.R b/local/tests/test-r-minor-helpers.R index d68e99b..ac7b0a6 100644 --- a/local/tests/test-r-minor-helpers.R +++ b/local/tests/test-r-minor-helpers.R @@ -2,15 +2,27 @@ source(file.path("..", "r-minor-helpers.R")) test_that("pure-r (NeedsCompilation != yes) is not sensitive", { expect_identical(classify_from_metadata("no", NA, c("Rcpp")), "not-sensitive") - expect_identical(classify_from_metadata("", "Rcpp", c("Rcpp")), "not-sensitive") + expect_identical( + classify_from_metadata("", "Rcpp", c("Rcpp")), + "not-sensitive" + ) }) test_that("LinkingTo a risky dep is sensitive (version constraints stripped)", { - expect_identical(classify_from_metadata("yes", "Rcpp (>= 1.0)", c("Rcpp")), "sensitive") - expect_identical(classify_from_metadata("yes", "R6,\n cpp11", c("Rcpp", "cpp11")), "sensitive") + expect_identical( + classify_from_metadata("yes", "Rcpp (>= 1.0)", c("Rcpp")), + "sensitive" + ) + expect_identical( + classify_from_metadata("yes", "R6,\n cpp11", c("Rcpp", "cpp11")), + "sensitive" + ) }) test_that("compiled but no risky LinkingTo is ambiguous (needs source)", { - expect_identical(classify_from_metadata("yes", "R6", c("Rcpp", "cpp11")), "ambiguous") + expect_identical( + classify_from_metadata("yes", "R6", c("Rcpp", "cpp11")), + "ambiguous" + ) expect_identical(classify_from_metadata("yes", NA, c("Rcpp")), "ambiguous") })