# Load necessary libraries
library(httr2)
library(dplyr)
library(tidyr)
library(xml2)
library(tools)
library(future)
library(future.apply)

# Define the base URL for the CRAN archive
base_url <- "https://cran.r-project.org/src/contrib/Archive/"

# Function to get all package names using tools::CRAN_package_db()
get_package_names <- function() {
  cran_db <- tools::CRAN_package_db()
  package_names <- unique(cran_db$Package)
  return(package_names)
}

# Function to get all archived versions of a package with error handling
get_archived_versions <- function(package_name) {
  package_url <- paste0(base_url, package_name, "/")
  
  # Try to perform the request and handle any errors
  tryCatch({
    response <- request(package_url) %>% req_perform()
    page <- response %>% resp_body_string() %>% read_html()
    version_links <- page %>% xml_find_all("//a") %>% xml_attr("href")
    # Extract version numbers
    versions <- gsub(paste0(package_name, "_|\\.tar\\.gz"), "", version_links)
    # Filter out invalid version strings
    versions <- versions[grep("^[0-9]+\\.[0-9]+\\.[0-9]+$", versions)]
    return(data.frame(Package = package_name, Version = versions, stringsAsFactors = FALSE))
  }, error = function(e) {
    # If there's an error (e.g., HTTP 404), return NULL
    return(NULL)
  })
}

# Get all package names
package_names <- get_package_names()

# Set up parallel processing
plan(multisession, workers = parallel::detectCores())

# Get archived versions for all packages in parallel
all_packages_df <- future_lapply(package_names, get_archived_versions) %>%
  bind_rows()

# Display the dataframe
print(all_packages_df)