devxy/build-cran-binaries
Watch
1
0
Fork
You've already forked build-cran-binaries
 0
Code Issues 10 Pull requests Releases Activity

use new s3 bucket

Browse source
This commit is contained in:
Patrick Schratz 2024-12-22 14:32:09 +01:00
commit ad768bce27
Signed by: pat-s
GPG key ID: 3C6318841EF78925
13 changed files with 102 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

2
docker/Containerfile-alpine-320
View file

@ -57,7 +57,7 @@ RUN bash -c 'echo -e "options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.gl
RUN bash -c 'echo -e "PKG_SYSREQS=TRUE\nPKG_SYSREQS_VERBOSE=TRUE" > ~/.Renviron'
### INSTALL R
RUN curl -O "https://devxy-r-builds.s3.eu-central-003.backblazeb2.com/alpine320/r-${R_VERSION}_1_$(arch).apk"
RUN curl -O "https://devxy-r-builds.s3.fsn1.backblazeb2.com/alpine320/r-${R_VERSION}_1_$(arch).apk"
RUN apk add --allow-untrusted "r-${R_VERSION}_1_$(arch).apk" && rm "r-${R_VERSION}_1_$(arch).apk"
RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'

2
docker/Containerfile-alpine-321
View file

@ -57,7 +57,7 @@ RUN bash -c 'echo -e "options(crayon.enabled = TRUE, Ncpus = ${NCPUS}, future.gl
RUN bash -c 'echo -e "PKG_SYSREQS=TRUE\nPKG_SYSREQS_VERBOSE=TRUE" > ~/.Renviron'
### INSTALL R
RUN curl -O "https://devxy-r-builds.s3.eu-central-003.backblazeb2.com/alpine321/r-${R_VERSION}_1_$(arch).apk"
RUN curl -O "https://devxy-r-builds.s3.fsn1.backblazeb2.com/alpine321/r-${R_VERSION}_1_$(arch).apk"
RUN apk add --allow-untrusted "r-${R_VERSION}_1_$(arch).apk" && rm "r-${R_VERSION}_1_$(arch).apk"
RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'

2
docker/Containerfile-redhat-8
View file

@ -43,7 +43,7 @@ RUN cd /JAGS-4.3.2 && make 2>&1 && make install 2>&1 >/dev/null && rm -rf JAGS-4
RUN dnf install -y chromium
### INSTALL R
RUN curl -O "https://devxy-r-builds.s3.eu-central-003.backblazeb2.com/el8/R-${R_VERSION}-1-1.$(arch).rpm"
RUN curl -O "https://devxy-r-builds.s3.fsn1.backblazeb2.com/el8/R-${R_VERSION}-1-1.$(arch).rpm"
RUN dnf install -q -y "R-${R_VERSION}-1-1.$(arch).rpm" && rm "R-${R_VERSION}-1-1.$(arch).rpm"
RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'

2
docker/Containerfile-redhat-9
View file

@ -43,7 +43,7 @@ RUN cd /JAGS-4.3.2 && make 2>&1 && make install 2>&1 >/dev/null && rm -rf JAGS-4
RUN dnf install -y chromium
### INSTALL R
RUN curl -O "https://devxy-r-builds.s3.eu-central-003.backblazeb2.com/el9/R-${R_VERSION}-1-1.$(arch).rpm"
RUN curl -O "https://devxy-r-builds.s3.fsn1.backblazeb2.com/el9/R-${R_VERSION}-1-1.$(arch).rpm"
RUN dnf install -q -y "R-${R_VERSION}-1-1.$(arch).rpm" && rm "R-${R_VERSION}-1-1.$(arch).rpm"
RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'

2
docker/Containerfile-ubuntu-2204
View file

@ -37,7 +37,7 @@ RUN apt-get install -y software-properties-common && \
apt install -y chromium
### INSTALL R
RUN curl -O "https://devxy-r-builds.s3.eu-central-003.backblazeb2.com/2204/r-${R_VERSION}_1_$(dpkg --print-architecture).deb"
RUN curl -O "https://devxy-r-builds.s3.fsn1.backblazeb2.com/2204/r-${R_VERSION}_1_$(dpkg --print-architecture).deb"
RUN gdebi -n -qq "r-${R_VERSION}_1_$(dpkg --print-architecture).deb" && rm "r-${R_VERSION}_1_$(dpkg --print-architecture).deb"
RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'

2
docker/Containerfile-ubuntu-2404
View file

@ -37,7 +37,7 @@ RUN apt-get install -y software-properties-common && \
apt install -y chromium
### INSTALL R
RUN curl -O "https://devxy-r-builds.s3.eu-central-003.backblazeb2.com/2404/r-${R_VERSION}_1_$(dpkg --print-architecture).deb"
RUN curl -O "https://devxy-r-builds.s3.fsn1.backblazeb2.com/2404/r-${R_VERSION}_1_$(dpkg --print-architecture).deb"
RUN gdebi -n -qq "r-${R_VERSION}_1_$(dpkg --print-architecture).deb" && rm "r-${R_VERSION}_1_$(dpkg --print-architecture).deb"
RUN R -q -e 'install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/devel/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))'

6
local/archive-missed-pkgs.R
View file

@ -3,14 +3,14 @@ library(bincraftR)
s3fs::s3_file_system(
aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
endpoint = "https://s3.eu-central-003.backblazeb2.com",
region_name = "eu-central-003",
endpoint = "fsn1.your-objectstorage.com",
region_name = "fsn1",
)
arch = "amd64"
os = "noble"
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/%s/%s/latest/src/contrib", arch, os))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/%s/%s/latest/src/contrib", arch, os))
length(unique(sapply(strsplit(basename(files), "_"), function(x) x[1])))
non_archived = unique(sapply(strsplit(basename(files), "_"), function(x) x[1])[duplicated(sapply(strsplit(basename(files), "_"), function(x) x[1]))])
# RInno is not built because it only exists for Windows

43
local/debug-packages-writing.R
View file

@ -1,4 +1,41 @@
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel9/latest/src/contrib"))
library(future)
future::plan(multisession)
future::plan(sequential)
time = Sys.time()
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/rhel9/latest/src/contrib/Archive"), recurse = T)
Sys.time() - time
archive <- llply(dirs, function(dir) {
files <- list.files(dir, recursive = FALSE, full.names = TRUE, pattern = "*.tar.gz")
if (length(files) == 0) {
print(paste0("Error: Empty directory: ", dir))
return(NULL)
}
info <- file.info(files)
tryCatch({
rownames(info) <- paste0(basename(dirname(files)), "/", basename(files))
}, error = function(e) {
print(paste0("Error: Exception catched for Archived directory: ", dir))
print(e)
return(NULL)
})
info
})
tryCatch({
rownames(info) <- paste0(basename(dirname(files)), "/", basename(files))
}, error = function(e) {
print(paste0("Error: Exception catched for Archived directory: ", dir))
print(e)
return(NULL)
})
curl::curl_download("https://cran.devxy.io/amd64/rhel9/latest/src/contrib/PACKAGES.db", "PACKAGES.db")
con = DBI::dbConnect(RSQLite::SQLite(), "PACKAGES.db")
@ -12,3 +49,7 @@ which(grepl("digest", df$Package))
system("cat /tmp/PACKAGES | grep '^Package: digest' | wc -l")
curl::curl_fetch_memory("https://cloud.r-project.org/src/contrib/Meta/archive.rds")
remotes = readRDS(url("https://cloud.r-project.org/src/contrib/Meta/archive.rds", "rb"))
packages = readRDS(url("https://cran.devxy.io/arm64/noble/latest/src/contrib/PACKAGES.rds", "rb"))

20
local/detect-duplicates.R
View file

@ -1,15 +1,15 @@
### Scope: Shows if there are duplicated packages at the S3 repository root (e.g. due to missed/aborted archiving)
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/alpine320/latest/src/contrib"))
files_b=basename(files)
files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])

7
local/missing-cran-packages-db.R
View file

@ -24,6 +24,7 @@ arch <- "amd64"
platform <- "redhat-8"
platform <- "redhat-9"
platform <- "alpine-320"
platform <- "alpine-321"
platform <- "ubuntu-2204"
platform <- "ubuntu-2404"
@ -33,10 +34,12 @@ pkgs_db <- unique(data$name)
pkgs_db = setdiff(pkgs_db, removed_pkgs)
pkgs <- setdiff(cran_pkgs, pkgs_db)
formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:900])))))
formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:length(pkgs)])))))
formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
cat(formatted_pkgs_one_line, "\n", sep = "")
length(formatted_pkgs)
length(pkgs)
clipr::write_clip(formatted_pkgs_one_line)
# sapply(pkgs, function(x) which(grepl(sprintf("^%s$", x), cran_pkgs)))

24
local/missing-cran-packages.R
View file

@ -4,21 +4,21 @@ library(dplyr)
s3fs::s3_file_system(
aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
endpoint = "https://s3.eu-central-003.backblazeb2.com",
region_name = "eu-central-003",
endpoint = "fsn1.your-objectstorage.com",
region_name = "fsn1",
)
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/jammy/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/noble/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel8/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel9/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/alpine320/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/jammy/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/noble/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/rhel8/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/rhel9/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/alpine320/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/jammy/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/noble/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel8/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel9/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/jammy/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/noble/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/rhel8/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/rhel9/latest/src/contrib")) # done
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/alpine320/latest/src/contrib"))
files_b=basename(files)
files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])

42
local/missing-packages-in-index.R
View file

@ -3,21 +3,21 @@ library(dplyr)
s3fs::s3_file_system(
aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
endpoint = "https://s3.eu-central-003.backblazeb2.com",
region_name = "eu-central-003",
endpoint = "fsn1.your-objectstorage.com",
region_name = "fsn1",
)
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/alpine320/latest/src/contrib"))
files_b=basename(files)
files_b_split = sapply(strsplit(basename(files_b), "_"), function(x) x[1])
@ -36,16 +36,16 @@ if (length(files_b_split) != length(pkgs_index_b)) {
cat(formatted_pkgs_one_line, "\n", sep = "")
}
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/amd64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/amd64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-arm64-r-binaries/arm64/alpine320/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/jammy/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/noble/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/rhel8/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/rhel9/latest/src/contrib"))
files <- s3fs::s3_dir_ls(sprintf("devxy-r-package-binaries/arm64/alpine320/latest/src/contrib"))

3
local/packages-without-any-binary.R
View file

@ -22,7 +22,8 @@ pkgs <- data |>
pkgs = setdiff(cran_pkgs, pkgs)
# Format, remove line breaks, and print as a single line
formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[1:300])))))
formatted_pkgs <- gsub('"', "'", capture.output(dput(as.character(na.omit(pkgs[100:900])))))
formatted_pkgs_one_line <- paste(formatted_pkgs, collapse = " ")
cat(formatted_pkgs_one_line, "\n", sep = "")