fix: save S3 listing in precompute step, avoid loading s3fs in build step
Some checks failed
ci/crow/manual/build-all-versions-install-deps-amd64 Pipeline was successful
ci/crow/manual/build-all-versions-install-deps-arm64 Pipeline was successful
ci/crow/manual/build-all-versions-arm64/2 Pipeline failed
ci/crow/cron/update-package-index-alpine-322-amd64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-322-arm64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-323-amd64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-323-arm64 Pipeline was successful
ci/crow/cron/update-package-index-redhat-8-amd64 Pipeline was successful
ci/crow/cron/update-package-index-redhat-8-arm64 Pipeline was successful
ci/crow/cron/update-package-index-redhat-9-amd64 Pipeline was successful
ci/crow/cron/update-package-index-redhat-9-arm64 Pipeline was successful
ci/crow/cron/update-package-index-ubuntu-2204-amd64 Pipeline was successful
ci/crow/cron/update-package-index-ubuntu-2204-arm64 Pipeline was successful
ci/crow/manual/build-all-versions-arm64/3 Pipeline failed
ci/crow/cron/update-package-index-ubuntu-2404-amd64 Pipeline was successful
ci/crow/cron/update-package-index-ubuntu-2404-arm64 Pipeline was successful
ci/crow/manual/build-all-versions-arm64/4 Pipeline failed
ci/crow/manual/build-all-versions-arm64/1 Pipeline failed
ci/crow/cron/process-updates-ubuntu-2404-arm64 Pipeline was successful
ci/crow/manual/build-all-versions-amd64/2 Pipeline failed
ci/crow/manual/build-all-versions-amd64/4 Pipeline failed
ci/crow/manual/build-all-versions-amd64/1 Pipeline failed
ci/crow/manual/build-all-versions-amd64/3 Pipeline failed
ci/crow/cron/process-updates-alpine-322-amd64 Pipeline was successful
ci/crow/cron/process-updates-redhat-8-amd64 Pipeline was successful
ci/crow/cron/process-updates-alpine-322-arm64 Pipeline was successful
ci/crow/cron/process-updates-redhat-8-arm64 Pipeline was successful
ci/crow/cron/process-updates-redhat-9-amd64 Pipeline was successful
ci/crow/cron/process-updates-alpine-323-amd64 Pipeline was successful
ci/crow/cron/process-updates-alpine-323-arm64 Pipeline was successful
ci/crow/cron/process-updates-redhat-9-arm64 Pipeline was successful
ci/crow/cron/process-updates-ubuntu-2204-amd64 Pipeline was successful
ci/crow/cron/process-updates-ubuntu-2204-arm64 Pipeline was successful
ci/crow/cron/process-updates-ubuntu-2404-amd64 Pipeline was successful
Some checks failed
ci/crow/manual/build-all-versions-install-deps-amd64 Pipeline was successful
ci/crow/manual/build-all-versions-install-deps-arm64 Pipeline was successful
ci/crow/manual/build-all-versions-arm64/2 Pipeline failed
ci/crow/cron/update-package-index-alpine-322-amd64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-322-arm64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-323-amd64 Pipeline was successful
ci/crow/cron/update-package-index-alpine-323-arm64 Pipeline was successful
ci/crow/cron/update-package-index-redhat-8-amd64 Pipeline was successful
ci/crow/cron/update-package-index-redhat-8-arm64 Pipeline was successful
ci/crow/cron/update-package-index-redhat-9-amd64 Pipeline was successful
ci/crow/cron/update-package-index-redhat-9-arm64 Pipeline was successful
ci/crow/cron/update-package-index-ubuntu-2204-amd64 Pipeline was successful
ci/crow/cron/update-package-index-ubuntu-2204-arm64 Pipeline was successful
ci/crow/manual/build-all-versions-arm64/3 Pipeline failed
ci/crow/cron/update-package-index-ubuntu-2404-amd64 Pipeline was successful
ci/crow/cron/update-package-index-ubuntu-2404-arm64 Pipeline was successful
ci/crow/manual/build-all-versions-arm64/4 Pipeline failed
ci/crow/manual/build-all-versions-arm64/1 Pipeline failed
ci/crow/cron/process-updates-ubuntu-2404-arm64 Pipeline was successful
ci/crow/manual/build-all-versions-amd64/2 Pipeline failed
ci/crow/manual/build-all-versions-amd64/4 Pipeline failed
ci/crow/manual/build-all-versions-amd64/1 Pipeline failed
ci/crow/manual/build-all-versions-amd64/3 Pipeline failed
ci/crow/cron/process-updates-alpine-322-amd64 Pipeline was successful
ci/crow/cron/process-updates-redhat-8-amd64 Pipeline was successful
ci/crow/cron/process-updates-alpine-322-arm64 Pipeline was successful
ci/crow/cron/process-updates-redhat-8-arm64 Pipeline was successful
ci/crow/cron/process-updates-redhat-9-amd64 Pipeline was successful
ci/crow/cron/process-updates-alpine-323-amd64 Pipeline was successful
ci/crow/cron/process-updates-alpine-323-arm64 Pipeline was successful
ci/crow/cron/process-updates-redhat-9-arm64 Pipeline was successful
ci/crow/cron/process-updates-ubuntu-2204-amd64 Pipeline was successful
ci/crow/cron/process-updates-ubuntu-2204-arm64 Pipeline was successful
ci/crow/cron/process-updates-ubuntu-2404-amd64 Pipeline was successful
s3fs uses reticulate/Python which significantly increases R process memory. When pak tries to fork R to install package dependencies, the enlarged process can't be duplicated within the container memory limit. Now the install-deps step saves the S3 file listing as s3_cache.rds, and the build step reads it with readRDS() — no s3fs loading needed.
This commit is contained in:
parent
bc3dbd6126
commit
8a98cf96a6
2 changed files with 8 additions and 15 deletions
|
|
@ -40,21 +40,10 @@ exclude <- c(
|
|||
)
|
||||
chunk <- chunk[!chunk$Package %in% exclude, ]
|
||||
|
||||
# Bulk S3 listing upfront — avoids per-package S3 calls
|
||||
s3fs::s3_file_system(
|
||||
aws_access_key_id = Sys.getenv("B2_S3_ACCESS_KEY"),
|
||||
aws_secret_access_key = Sys.getenv("B2_S3_SECRET_KEY"),
|
||||
endpoint = "https://s3.eu-central-003.backblazeb2.com",
|
||||
region_name = "eu-central-003",
|
||||
refresh = TRUE
|
||||
)
|
||||
codename <- bincraft::set_codename(NULL)
|
||||
local_arch <- Sys.info()[["machine"]]
|
||||
arch <- if (grepl("arm64|aarch64", local_arch)) "arm64" else "amd64"
|
||||
s3_cache <- basename(s3fs::s3_dir_ls(
|
||||
sprintf("devxy-rpkgs-binaries/%s/%s/latest/src/contrib", arch, codename),
|
||||
recurse = TRUE
|
||||
))
|
||||
# Read pre-computed S3 listing from install-deps step
|
||||
# This avoids loading s3fs/reticulate/Python in the build container,
|
||||
# saving significant memory for pak subprocess forks
|
||||
s3_cache <- readRDS("/mnt/cache/packages/s3_cache.rds")
|
||||
sprintf("S3 cache: %s files", length(s3_cache))
|
||||
|
||||
n <- nrow(chunk)
|
||||
|
|
|
|||
|
|
@ -83,6 +83,10 @@ s3_pkgs = s3fs::s3_dir_ls(
|
|||
recurse = TRUE
|
||||
)
|
||||
|
||||
# Save the raw S3 file listing for the build step to use as s3_package_cache
|
||||
# This avoids loading s3fs/reticulate in the build container, saving memory for pak forks
|
||||
saveRDS(basename(s3_pkgs), "/mnt/cache/packages/s3_cache.rds")
|
||||
|
||||
file_names <- basename(s3_pkgs)
|
||||
matches <- regexec("^([A-Za-z0-9.]+)_([0-9][^/]*)\\.tar\\.gz$", file_names)
|
||||
parts <- regmatches(file_names, matches)
|
||||
|
|
|
|||
Loading…
Reference in a new issue